[ Jun 20 21:23:24 Enabled. ] [ Jun 20 21:23:24 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Jun 20 21:23:24 Method "start" exited with status 0. ] [ Jun 20 21:23:24 Rereading configuration. ] [ Jun 20 21:23:24 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:24.194672884Z","hostname":"sock","pid":1555,"file":"illumos-utils/src/running_zone.rs:163"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:24.194854369Z","hostname":"sock","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:24.24725701Z","hostname":"sock","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:24.247303971Z","hostname":"sock","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:24.359181723Z","hostname":"sock","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:24.36809142Z","hostname":"sock","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:24.375546139Z","hostname":"sock","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:24.97627405Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fdb0:18c0:4d0d:a02a::, len: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.1179992Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:119"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.118051992Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:131"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.118118023Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:141"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.118205306Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:162","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.118224876Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:506"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.118245147Z","hostname":"sock","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:625"} {"msg":"Generated i86pc baseboard Pc { identifier: \"sock\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.121203472Z","hostname":"sock","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:535"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:25.121210352Z","hostname":"sock","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:674"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.121222622Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:176"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.121228372Z","hostname":"sock","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:733"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.121254623Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.121305034Z","hostname":"sock","pid":1555,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:262","disks":"{}","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.121368706Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.121381946Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.121404857Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.133636238Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.133681899Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:315","id":"d4a00029-4475-4b78-929a-0a3975ebfb87 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.233860954Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: d4a00029-4475-4b78-929a-0a3975ebfb87 (zpool), kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.441044967Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.441086558Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.441092778Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.441098649Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.441113139Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.441117939Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.44113717Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.450138188Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.450167879Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:315","id":"10b67551-9fe8-4a7e-aee1-8eed094ef18b (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.538060262Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 10b67551-9fe8-4a7e-aee1-8eed094ef18b (zpool), kind: Internal }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.74071138Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_0.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.740749381Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: d4a00029-4475-4b78-929a-0a3975ebfb87 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 10b67551-9fe8-4a7e-aee1-8eed094ef18b (zpool), kind: Internal } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.740772002Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.740784692Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.740794503Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.740798383Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.740804923Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_1.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.740855314Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: d4a00029-4475-4b78-929a-0a3975ebfb87 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 10b67551-9fe8-4a7e-aee1-8eed094ef18b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.740869234Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.740873385Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.740877845Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.740880695Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.740909306Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_2.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.740919046Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: d4a00029-4475-4b78-929a-0a3975ebfb87 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 10b67551-9fe8-4a7e-aee1-8eed094ef18b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.740949567Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.740959277Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.740966607Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.740975117Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.740981257Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_3.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.740987627Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: d4a00029-4475-4b78-929a-0a3975ebfb87 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 10b67551-9fe8-4a7e-aee1-8eed094ef18b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.741012608Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.741019818Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.741030798Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.741036729Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.741043969Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_4.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.741052719Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.741058659Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.741065479Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: d4a00029-4475-4b78-929a-0a3975ebfb87 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 10b67551-9fe8-4a7e-aee1-8eed094ef18b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.74110117Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.741108891Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.741115141Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_5.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.741122831Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: d4a00029-4475-4b78-929a-0a3975ebfb87 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 10b67551-9fe8-4a7e-aee1-8eed094ef18b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.741151621Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.741161342Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.741168682Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.741174752Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.741183122Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_6.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.741190602Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.741196453Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.741206783Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: d4a00029-4475-4b78-929a-0a3975ebfb87 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 10b67551-9fe8-4a7e-aee1-8eed094ef18b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.741242674Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.741250374Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.741256384Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_7.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.741266544Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: d4a00029-4475-4b78-929a-0a3975ebfb87 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 10b67551-9fe8-4a7e-aee1-8eed094ef18b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.741294615Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.741301995Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.741308996Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.741318846Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.741326516Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_8.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.741336516Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.741352607Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.741359937Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: d4a00029-4475-4b78-929a-0a3975ebfb87 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 10b67551-9fe8-4a7e-aee1-8eed094ef18b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.741392408Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.741399638Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.741405848Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:91"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.741421828Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:93"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.741428348Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: d4a00029-4475-4b78-929a-0a3975ebfb87 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 10b67551-9fe8-4a7e-aee1-8eed094ef18b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.7414793Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.74148794Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.741495Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.74150439Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:200"} {"msg":"No ledger in /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.741511171Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.741519541Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.741526501Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.741553992Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.741561372Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.741572302Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.741580812Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:205"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.741588893Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:220"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.752751716Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:711"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.752794767Z","hostname":"sock","pid":1555,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:289","time_to_next_cleanup":"599.999923568s","next_cleanup":"Instant { tv_sec: 1212, tv_nsec: 315617151 }"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.752858609Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3647"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.752869839Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.752939931Z","hostname":"sock","pid":1555,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.752950351Z","hostname":"sock","pid":1555,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.756102511Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:196"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.756359208Z","hostname":"sock","pid":1555,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:44","local_addr":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.2.164,\n last: 10.151.2.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.2.164,\n 10.151.2.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.2.174,\n infra_ip_last: 10.151.2.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.2.100,\n vlan_id: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.2.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.756609914Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1011"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.756618974Z","hostname":"sock","pid":1555,"component":"DnsResolver","component":"RSS","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.756678846Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.756730907Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.756754038Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.756758428Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1071"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.756786999Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.756809559Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.756818339Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.7568358Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1112"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.75683984Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:156"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:a02a::1","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.75684344Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:129"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.756852091Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.756880491Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.75721368Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Sled plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.757371914Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:176"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.757386274Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1145"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.757436955Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.757443975Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.75761143Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.757747613Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.757783874Z","hostname":"sock","pid":1555,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:217","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 8b6fc4f9-f2be-4116-a97c-1830a2193641, rack_id: 0aedcb47-b77d-4cb1-8181-dc0c6aa83175, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.757898677Z","hostname":"sock","pid":1555,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:59246","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:66"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 8b6fc4f9-f2be-4116-a97c-1830a2193641, rack_id: 0aedcb47-b77d-4cb1-8181-dc0c6aa83175, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.75802351Z","hostname":"sock","pid":1555,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:360"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.75803225Z","hostname":"sock","pid":1555,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:375"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.758057961Z","hostname":"sock","pid":1555,"file":"sled-agent/src/server.rs:44"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.758064481Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.758068121Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:530"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.758071591Z","hostname":"sock","pid":1555,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.758079822Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:101::, len: 64 }"} {"msg":"No ledger in /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.758089932Z","hostname":"sock","pid":1555,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.758105482Z","hostname":"sock","pid":1555,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.758146293Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:519"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.758150624Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:547"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.760875743Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3879"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.768152367Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.777551536Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.788302079Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.797562215Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.806097862Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.815035078Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.823230307Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.830788809Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.838419283Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"No disks to be automatically managed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.838470424Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:558"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.838480024Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:370"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.838487254Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:339"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.838492565Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:449","result":"DisksManagementResult { status: [] }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.838498045Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.857171209Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.875187207Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.890835924Z","hostname":"sock","pid":1555,"sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:354"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.890886836Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.890891866Z","hostname":"sock","pid":1555,"sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:366"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.890952537Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 67.109885ms","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.894698053Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d26f3a4c-8485-4549-83ed-4c4ab0c6f75b\", \"content-length\": \"124\", \"date\": \"Thu, 20 Jun 2024 21:23:24 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d26f3a4c-8485-4549-83ed-4c4ab0c6f75b\" }"} {"msg":"attempting to create encrypted zvol: name=\"oxi_d4a00029-4475-4b78-929a-0a3975ebfb87/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.904410219Z","hostname":"sock","pid":1555,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"successfully created encrypted zvol: name=\"oxi_d4a00029-4475-4b78-929a-0a3975ebfb87/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.928678496Z","hostname":"sock","pid":1555,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_d4a00029-4475-4b78-929a-0a3975ebfb87/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.928723227Z","hostname":"sock","pid":1555,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.934193206Z","hostname":"sock","pid":1555,"sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:381"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.934200846Z","hostname":"sock","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:126"} {"msg":"Ensuring dataset oxi_d4a00029-4475-4b78-929a-0a3975ebfb87/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.934205676Z","hostname":"sock","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:136"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.964207698Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 111.822234ms","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:25.965095631Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"12683b8a-5bef-4bbd-b868-f2669be2cdd9\", \"content-length\": \"124\", \"date\": \"Thu, 20 Jun 2024 21:23:24 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"12683b8a-5bef-4bbd-b868-f2669be2cdd9\" }"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:26.005543698Z","hostname":"sock","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:170"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:26.07764398Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 157.897771ms","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:26.078270086Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fe828b16-4cf7-4e8c-aedd-0f65647eeddf\", \"content-length\": \"124\", \"date\": \"Thu, 20 Jun 2024 21:23:26 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"fe828b16-4cf7-4e8c-aedd-0f65647eeddf\" }"} {"msg":"Mounting oxi_d4a00029-4475-4b78-929a-0a3975ebfb87/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:26.22447406Z","hostname":"sock","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:178"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:26.236760132Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 190.846882ms","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:26.237360318Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"50645127-ffee-49e6-b7fa-894631af4f42\", \"content-length\": \"124\", \"date\": \"Thu, 20 Jun 2024 21:23:26 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"50645127-ffee-49e6-b7fa-894631af4f42\" }"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:26.237757108Z","hostname":"sock","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:26.23784189Z","hostname":"sock","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:26.237873211Z","hostname":"sock","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:26.237888881Z","hostname":"sock","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:192"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:26.431071079Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 116.643765ms","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:26.432198797Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"77284e2a-c372-4146-8ce5-b5ff5e41e22f\", \"content-length\": \"124\", \"date\": \"Thu, 20 Jun 2024 21:23:26 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"77284e2a-c372-4146-8ce5-b5ff5e41e22f\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:26.549915528Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 159.891976ms","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:26.550745339Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"cab7a4eb-1db2-4d52-bbcc-6e04de9da48f\", \"content-length\": \"124\", \"date\": \"Thu, 20 Jun 2024 21:23:26 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"cab7a4eb-1db2-4d52-bbcc-6e04de9da48f\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:26.711808461Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 346.273957ms","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:26.712337345Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"384aff10-e857-4d16-a0b2-03bbe6c57392\", \"content-length\": \"124\", \"date\": \"Thu, 20 Jun 2024 21:23:26 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"384aff10-e857-4d16-a0b2-03bbe6c57392\" }"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:26.963879355Z","hostname":"sock","pid":1555,"sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:90"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:26.96446845Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:39768","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:196","component":"producer-server","component":"MetricsManager","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:196"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:26.96449476Z","hostname":"sock","pid":1555,"component":"internal-dns-resolver","file":"internal-dns/src/resolver.rs:60","component":"producer-server","component":"MetricsManager","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:26.964543232Z","hostname":"sock","pid":1555,"file":"oximeter/producer/src/lib.rs:276","component":"producer-server","component":"MetricsManager","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"oximeter/producer/src/lib.rs:276","interval":"30s","address":"[fd00:1122:3344:101::1]:39768","producer_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:26.965984978Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"299.265297ms"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:27.00934794Z","hostname":"sock","pid":1555,"component":"VmmReservoirManager","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:27.009387811Z","hostname":"sock","pid":1555,"component":"VmmReservoirManager","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:27.009466323Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:993","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:27.009476413Z","hostname":"sock","pid":1555,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:27.010216992Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:196"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:27.010246553Z","hostname":"sock","pid":1555,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:430"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:27.010265673Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:27.010287384Z","hostname":"sock","pid":1555,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:27.010504809Z","hostname":"sock","pid":1555,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:27.010672354Z","hostname":"sock","pid":1555,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:27.010689384Z","hostname":"sock","pid":1555,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:262","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV2 { rack_subnet: Ipv6Net { addr: fd00:1122:3344:100::, width: 56 }, infra_ip_first: 10.151.2.174, infra_ip_last: 10.151.2.174, ports: [PortConfigV2 { routes: [RouteConfig { destination: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), nexthop: 10.151.2.100, vlan_id: None }], addresses: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false }], bgp: [], bfd: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:27.010704014Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3647"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:27.010730805Z","hostname":"sock","pid":1555,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:59246","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:73"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:27.215656451Z","hostname":"sock","pid":1555,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:108","peer":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:27.215751994Z","hostname":"sock","pid":1555,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:27.215858206Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:27.215944799Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"No ledger in /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:27.215961399Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:27.21598828Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:27.216040001Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:27.216055842Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/8b6fc4f9-f2be-4116-a97c-1830a2193641): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:27.222638209Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:27.251743548Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:42270"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:27.251911922Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:27.251988204Z","hostname":"sock","pid":1555,"uri":"/inventory","method":"GET","req_id":"101f0479-0a2d-44f4-9631-d70cf449b536","remote_addr":"[fd00:1122:3344:101::1]:42270","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":148,"response_code":"200"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:27.267904589Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"519.23882ms"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:27.281375321Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:39662"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:27.281516234Z","hostname":"sock","pid":1555,"uri":"/sled-role","method":"GET","req_id":"6a48cdaf-411e-48be-894e-9d24f08f8615","remote_addr":"[fd00:1122:3344:101::1]:39662","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":48,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:27.282640473Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:27.282667484Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:27.283026743Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:27.283258749Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:812"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { disks: [OmicronPhysicalDiskConfig { id: 68781d25-6826-432c-848f-7622266d8e8a, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, pool_id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool) }, OmicronPhysicalDiskConfig { id: 26005838-e763-4213-9d78-5e23f5204772, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, pool_id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool) }, OmicronPhysicalDiskConfig { id: 57457066-9d86-401d-a52c-968b79b73715, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, pool_id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool) }, OmicronPhysicalDiskConfig { id: 1383ff3c-3136-464b-88d2-2dd336293ee1, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, pool_id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool) }, OmicronPhysicalDiskConfig { id: 82c35ad5-39a7-4795-9d05-30c678c1ba61, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, pool_id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool) }, OmicronPhysicalDiskConfig { id: 3a329bad-0c7d-4523-9b84-35f65c3ab172, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, pool_id: c3976b41-1b46-4fb3-86b8-14634aafccad (zpool) }, OmicronPhysicalDiskConfig { id: 37649d10-1fb2-45d4-9a9c-512a5a174bf3, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, pool_id: dd582fd3-6899-4123-a5b6-2a268c4ed13b (zpool) }, OmicronPhysicalDiskConfig { id: 4b9520bf-192f-4069-b8ae-600a5f1da316, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, pool_id: a553f0be-01ff-4b59-9e74-27e6ddf33491 (zpool) }, OmicronPhysicalDiskConfig { id: e19dfc3b-f82f-4331-a624-e151dca35b4b, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, pool_id: ff7b7614-aeaa-4dfa-8c5e-e5115ba48aa0 (zpool) }], generation: Generation(1) }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:27.311837625Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:348"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:27.312005939Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:47165"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 68781d25-6826-432c-848f-7622266d8e8a, pool_id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: 26005838-e763-4213-9d78-5e23f5204772, pool_id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: 57457066-9d86-401d-a52c-968b79b73715, pool_id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 1383ff3c-3136-464b-88d2-2dd336293ee1, pool_id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 82c35ad5-39a7-4795-9d05-30c678c1ba61, pool_id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: 3a329bad-0c7d-4523-9b84-35f65c3ab172, pool_id: c3976b41-1b46-4fb3-86b8-14634aafccad (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 37649d10-1fb2-45d4-9a9c-512a5a174bf3, pool_id: dd582fd3-6899-4123-a5b6-2a268c4ed13b (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 4b9520bf-192f-4069-b8ae-600a5f1da316, pool_id: a553f0be-01ff-4b59-9e74-27e6ddf33491 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: e19dfc3b-f82f-4331-a624-e151dca35b4b, pool_id: ff7b7614-aeaa-4dfa-8c5e-e5115ba48aa0 (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:27.312388699Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:27.312407759Z","hostname":"sock","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:27.31241579Z","hostname":"sock","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:27.31242249Z","hostname":"sock","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:681"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:27.312500862Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:339"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:27.312509062Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:27.312516752Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:27.312523062Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:27.426148989Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:27.42618483Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"68941de4-9d96-43c2-b5bb-c76900595397 (zpool)"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:27.844513488Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"859.746432ms"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8b6fc4f9-f2be-4116-a97c-1830a2193641): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-20T21:23:28.30358039Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:28.512275562Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:28.705715086Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:28.705848639Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:28.70586203Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:28.70587206Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:28.70587813Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:28.706096246Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:28.706106316Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:28.919801655Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"1.502065302s"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:29.11838252Z","hostname":"sock","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8b6fc4f9-f2be-4116-a97c-1830a2193641): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-20T21:23:29.796792795Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:29.796913138Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/8b6fc4f9-f2be-4116-a97c-1830a2193641): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:29.797048881Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 43.259243ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:30.009925229Z","hostname":"sock","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Automatically destroying dataset: oxp_68941de4-9d96-43c2-b5bb-c76900595397/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:30.205957849Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 72.091449ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:30.40006904Z","hostname":"sock","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:31.057882142Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"3.843815696s"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8b6fc4f9-f2be-4116-a97c-1830a2193641): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-20T21:23:31.058062797Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 188.345133ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:31.058712483Z","hostname":"sock","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 229.274911ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:31.65845648Z","hostname":"sock","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/8b6fc4f9-f2be-4116-a97c-1830a2193641): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:31.660134372Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 994.811595ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:32.480314949Z","hostname":"sock","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8b6fc4f9-f2be-4116-a97c-1830a2193641): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-20T21:23:32.48035912Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Looking for unencrypted datasets in oxp_68941de4-9d96-43c2-b5bb-c76900595397","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:32.928788922Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:33.13060945Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:33.130678331Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:33.130687232Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:33.130694632Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:33.130701122Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:33.330241051Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:33.330278382Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"c28d26f6-634d-4800-9bba-2148c8387fee (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:34.410097625Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:34.631185672Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:34.631232143Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:34.631238463Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:34.631263044Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:34.631276144Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:34.631406007Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:34.631416817Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:35.290320447Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"9.256601519s"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8b6fc4f9-f2be-4116-a97c-1830a2193641): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-20T21:23:35.742146766Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/8b6fc4f9-f2be-4116-a97c-1830a2193641): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:35.742209357Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:35.956724927Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_c28d26f6-634d-4800-9bba-2148c8387fee/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:36.419091493Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/8b6fc4f9-f2be-4116-a97c-1830a2193641): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:37.119564729Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:37.322231508Z","hostname":"sock","pid":1555,"component":"VmmReservoirManager","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8b6fc4f9-f2be-4116-a97c-1830a2193641): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-20T21:23:37.324246309Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Looking for unencrypted datasets in oxp_c28d26f6-634d-4800-9bba-2148c8387fee","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:37.431016241Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:37.438731907Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:37.438761958Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:37.438768978Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:37.438773869Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:37.438779958Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:37.448513186Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:37.448544297Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:37.542815532Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","zpool":"ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:37.551050571Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:37.551084372Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:37.551112572Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:37.551124313Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:37.551132863Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:37.551298437Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:37.551308727Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:37.585914007Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_3dcbe67c-4d44-4c02-82ee-c644a9177203/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:37.594090084Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_3dcbe67c-4d44-4c02-82ee-c644a9177203","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:37.716874784Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:37.724485797Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:37.724523648Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:37.724532498Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:37.724538258Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:37.724543618Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:37.734197174Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:37.734225815Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:37.828959101Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","zpool":"ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:37.837236812Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:37.837289353Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:37.837350024Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:37.837398236Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:37.837412126Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:37.837543869Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:37.83755174Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:37.871558273Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_ea2c3cdd-ebf6-434c-a220-e4b6e1180947/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:37.87889497Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_ea2c3cdd-ebf6-434c-a220-e4b6e1180947","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:37.999658918Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.007893187Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.007942518Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.007949928Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.007954738Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.007958998Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.018079726Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.018109076Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.111446618Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","zpool":"ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.119838951Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.119873292Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.119882182Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.119892982Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.119903382Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.120034036Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.120044976Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.15524607Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_05b3cfac-441e-4e11-8952-0d7b0cba33c0/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.16308125Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_05b3cfac-441e-4e11-8952-0d7b0cba33c0","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.287711996Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.295670898Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.295722799Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.295731029Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.295736889Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.29574209Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.305435366Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.305475077Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"c3976b41-1b46-4fb3-86b8-14634aafccad (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.402681017Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","zpool":"ZpoolName { id: c3976b41-1b46-4fb3-86b8-14634aafccad (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.411206143Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.411266735Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.411275975Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.411286695Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.411297385Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.41146465Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.4114826Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8b6fc4f9-f2be-4116-a97c-1830a2193641): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-20T21:23:38.415099172Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.446203452Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_c3976b41-1b46-4fb3-86b8-14634aafccad/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.453887747Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_c3976b41-1b46-4fb3-86b8-14634aafccad","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.578900413Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.586765983Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.586801974Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.586816504Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.586821195Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.586825105Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.596664735Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.596684695Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"dd582fd3-6899-4123-a5b6-2a268c4ed13b (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.692184551Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","zpool":"ZpoolName { id: dd582fd3-6899-4123-a5b6-2a268c4ed13b (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.698930133Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.698956363Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.698961813Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.698982614Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.699036475Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.699198439Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.6992155Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/8b6fc4f9-f2be-4116-a97c-1830a2193641): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:38.702340519Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.73465504Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_dd582fd3-6899-4123-a5b6-2a268c4ed13b/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.741627157Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Ensuring tfportrear0_0/ll exists in zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.791588897Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2333"} {"msg":"Looking for unencrypted datasets in oxp_dd582fd3-6899-4123-a5b6-2a268c4ed13b","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.863890414Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.872171494Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.872217395Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.872224155Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.872230275Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.872235626Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.881809459Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.88184623Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"a553f0be-01ff-4b59-9e74-27e6ddf33491 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.97677751Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","zpool":"ZpoolName { id: a553f0be-01ff-4b59-9e74-27e6ddf33491 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.984390254Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.984431215Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.984439645Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.984466016Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.984477876Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.98462402Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:38.98463589Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.018915261Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_a553f0be-01ff-4b59-9e74-27e6ddf33491/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.027305904Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_a553f0be-01ff-4b59-9e74-27e6ddf33491","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.15035765Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.157961633Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.158002934Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.158015245Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.158041366Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.158047006Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.167245519Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.1672767Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"ff7b7614-aeaa-4dfa-8c5e-e5115ba48aa0 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.264266044Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","zpool":"ZpoolName { id: ff7b7614-aeaa-4dfa-8c5e-e5115ba48aa0 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.271796625Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.271882978Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.271895368Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.271917779Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.271928159Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.272093283Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.272106953Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.306029805Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_ff7b7614-aeaa-4dfa-8c5e-e5115ba48aa0/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.313414753Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8b6fc4f9-f2be-4116-a97c-1830a2193641): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-20T21:23:39.417033785Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Looking for unencrypted datasets in oxp_ff7b7614-aeaa-4dfa-8c5e-e5115ba48aa0","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.432449857Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.440110441Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.440164453Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Writing ledger to /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.440225824Z","hostname":"sock","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.440389099Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: d4a00029-4475-4b78-929a-0a3975ebfb87 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 10b67551-9fe8-4a7e-aee1-8eed094ef18b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 }, zpool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 }, zpool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 }, zpool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 }, zpool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 }, zpool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 }, zpool_name: ZpoolName { id: c3976b41-1b46-4fb3-86b8-14634aafccad (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 }, zpool_name: ZpoolName { id: dd582fd3-6899-4123-a5b6-2a268c4ed13b (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033 }, zpool_name: ZpoolName { id: a553f0be-01ff-4b59-9e74-27e6ddf33491 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034 }, zpool_name: ZpoolName { id: ff7b7614-aeaa-4dfa-8c5e-e5115ba48aa0 (zpool), kind: External } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.44045484Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.44046245Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Writing ledger to /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.440577684Z","hostname":"sock","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.440876921Z","hostname":"sock","pid":1555,"uri":"/omicron-physical-disks","method":"PUT","req_id":"a026a16b-471c-4463-a981-6bdcf1b396ee","remote_addr":"[fd00:1122:3344:101::1]:47165","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":12128738,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: f4d810c4-7125-4f8d-90af-608014da1d8f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: eaab5753-bf35-45e9-8e6d-201b35af6027, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.47076036Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.470937765Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:44532"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.471643203Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.471647983Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.520665488Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.520705859Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.573733306Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.573794008Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.622967877Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.623014598Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.623048019Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.623103401Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.667108119Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.66716057Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.66717106Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_eaab5753-bf35-45e9-8e6d-201b35af6027","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.716822702Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_2da10309-7a29-4b33-90f3-4d4ffabdaad7","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.720416413Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_f4d810c4-7125-4f8d-90af-608014da1d8f","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.723995374Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_internal_dns_eaab5753-bf35-45e9-8e6d-201b35af6027","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.736676116Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_2da10309-7a29-4b33-90f3-4d4ffabdaad7","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.740968165Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_f4d810c4-7125-4f8d-90af-608014da1d8f","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:39.744960776Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8b6fc4f9-f2be-4116-a97c-1830a2193641): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-20T21:23:40.418538608Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/8b6fc4f9-f2be-4116-a97c-1830a2193641): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:40.70203062Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Profile for oxz_internal_dns_f4d810c4-7125-4f8d-90af-608014da1d8f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:41.290776517Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:41.290822368Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:1::, len: 64 }"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:41.291096705Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_f4d810c4-7125-4f8d-90af-608014da1d8f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_internal_dns_eaab5753-bf35-45e9-8e6d-201b35af6027:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:41.774818714Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_internal_dns_2da10309-7a29-4b33-90f3-4d4ffabdaad7:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:42.253794482Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:42.253873824Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:2::, len: 64 }"} {"msg":"wait for service Some(\"oxz_internal_dns_f4d810c4-7125-4f8d-90af-608014da1d8f\") failed: Property not found. retry in 62.848996ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:42.260957934Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_f4d810c4-7125-4f8d-90af-608014da1d8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:42.261002435Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:3::, len: 64 }"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:42.261093478Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_eaab5753-bf35-45e9-8e6d-201b35af6027","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:42.264693499Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_2da10309-7a29-4b33-90f3-4d4ffabdaad7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_internal_dns_f4d810c4-7125-4f8d-90af-608014da1d8f\") failed: Property not found. retry in 120.055557ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:42.333127778Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_f4d810c4-7125-4f8d-90af-608014da1d8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_eaab5753-bf35-45e9-8e6d-201b35af6027\") failed: Property not found. retry in 62.286361ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:42.382338308Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_eaab5753-bf35-45e9-8e6d-201b35af6027","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2da10309-7a29-4b33-90f3-4d4ffabdaad7\") failed: Property not found. retry in 42.376559ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:42.388416662Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_2da10309-7a29-4b33-90f3-4d4ffabdaad7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2da10309-7a29-4b33-90f3-4d4ffabdaad7\") failed: Property not found. retry in 74.174801ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:42.437604922Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_2da10309-7a29-4b33-90f3-4d4ffabdaad7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_eaab5753-bf35-45e9-8e6d-201b35af6027\") failed: Property not found. retry in 116.738775ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:42.560050453Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_eaab5753-bf35-45e9-8e6d-201b35af6027","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2da10309-7a29-4b33-90f3-4d4ffabdaad7\") failed: Property not found. retry in 113.623126ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:42.610047033Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_2da10309-7a29-4b33-90f3-4d4ffabdaad7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_f4d810c4-7125-4f8d-90af-608014da1d8f\") failed: Property not found. retry in 207.780491ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:42.698961232Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_f4d810c4-7125-4f8d-90af-608014da1d8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_eaab5753-bf35-45e9-8e6d-201b35af6027\") failed: Property not found. retry in 157.616332ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:42.734010582Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_eaab5753-bf35-45e9-8e6d-201b35af6027","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2da10309-7a29-4b33-90f3-4d4ffabdaad7\") failed: Property not found. retry in 570.58898ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:42.756235457Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_2da10309-7a29-4b33-90f3-4d4ffabdaad7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_eaab5753-bf35-45e9-8e6d-201b35af6027\") failed: Property not found. retry in 505.446162ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:42.903427946Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_eaab5753-bf35-45e9-8e6d-201b35af6027","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_f4d810c4-7125-4f8d-90af-608014da1d8f\") failed: Property not found. retry in 588.935655ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:42.916678783Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_f4d810c4-7125-4f8d-90af-608014da1d8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2da10309-7a29-4b33-90f3-4d4ffabdaad7\") failed: Property not found. retry in 585.206686ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:43.336234512Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_2da10309-7a29-4b33-90f3-4d4ffabdaad7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_eaab5753-bf35-45e9-8e6d-201b35af6027\") failed: Property not found. retry in 723.970888ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:43.417681651Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_eaab5753-bf35-45e9-8e6d-201b35af6027","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_f4d810c4-7125-4f8d-90af-608014da1d8f\") failed: Property not found. retry in 996.371059ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:43.51567906Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_f4d810c4-7125-4f8d-90af-608014da1d8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2da10309-7a29-4b33-90f3-4d4ffabdaad7\") failed: Property not found. retry in 1.023262616s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:43.932076139Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_2da10309-7a29-4b33-90f3-4d4ffabdaad7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_eaab5753-bf35-45e9-8e6d-201b35af6027\") failed: Property not found. retry in 981.717084ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:44.154965481Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_eaab5753-bf35-45e9-8e6d-201b35af6027","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_f4d810c4-7125-4f8d-90af-608014da1d8f\") failed: Property not found. retry in 1.436177658s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:44.521973085Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_f4d810c4-7125-4f8d-90af-608014da1d8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring bootstrap address fdb0:18c0:4d0d:a02a::2 exists in switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:44.85714627Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2350"} {"msg":"Adding bootstrap address","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:44.857162641Z","hostname":"sock","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:626"} {"msg":"wait for service Some(\"oxz_internal_dns_2da10309-7a29-4b33-90f3-4d4ffabdaad7\") failed: Property not found. retry in 553.296093ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:44.96379106Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_2da10309-7a29-4b33-90f3-4d4ffabdaad7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_eaab5753-bf35-45e9-8e6d-201b35af6027\") failed: Property not found. retry in 1.348623614s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:45.145501846Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_eaab5753-bf35-45e9-8e6d-201b35af6027","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Forwarding bootstrap traffic via oxBootstrap0 to fe80::8:20ff:fe20:d275","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:45.398005331Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2357"} {"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:45.403683225Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2418"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:45.522698859Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2603"} {"msg":"wait for service Some(\"oxz_internal_dns_2da10309-7a29-4b33-90f3-4d4ffabdaad7\") failed: Property not found. retry in 1.149669219s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:45.525455178Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_2da10309-7a29-4b33-90f3-4d4ffabdaad7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:45.759520855Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2767"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:45.948767603Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2482"} {"msg":"wait for service Some(\"oxz_internal_dns_f4d810c4-7125-4f8d-90af-608014da1d8f\") failed: Property not found. retry in 573.882132ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:45.969102108Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_f4d810c4-7125-4f8d-90af-608014da1d8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:46.250556579Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2517"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:46.466039323Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2817"} {"msg":"wait for service Some(\"oxz_internal_dns_eaab5753-bf35-45e9-8e6d-201b35af6027\") failed: Property not found. retry in 772.268658ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:46.502938881Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_eaab5753-bf35-45e9-8e6d-201b35af6027","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_f4d810c4-7125-4f8d-90af-608014da1d8f\") failed: Property not found. retry in 1.484468033s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:46.553439274Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_f4d810c4-7125-4f8d-90af-608014da1d8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:46.652262084Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2844"} {"msg":"wait for service Some(\"oxz_internal_dns_2da10309-7a29-4b33-90f3-4d4ffabdaad7\") failed: Property not found. retry in 682.806212ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:46.683225541Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_2da10309-7a29-4b33-90f3-4d4ffabdaad7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:46.876354097Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2711"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:47.067797501Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2511"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:47.078102633Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3903","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:47.078144504Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3919"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:47.078154424Z","hostname":"sock","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"wait for service Some(\"oxz_internal_dns_eaab5753-bf35-45e9-8e6d-201b35af6027\") failed: Property not found. retry in 1.23511236s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:47.285791669Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_eaab5753-bf35-45e9-8e6d-201b35af6027","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:47.641017454Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3927"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:47.648689238Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3983"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:47.794076492Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4045"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:47.894077683Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4026","rack_subnet":"fd00:1122:3344::"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:47.915312072Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4073"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.011860415Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4100"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.131240737Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:369"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.162643375Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:417"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.194791022Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:656"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.194875534Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:694"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.195376627Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:697","uptime":"3"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.195391677Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:439","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.2.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n}"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.271273955Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.271377738Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.271387858Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.271396718Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.271404948Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.271415009Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.271424929Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.271433159Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.271442559Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.27145043Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.27145843Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.27146678Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.27147664Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.27148482Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.271505051Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.271511511Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.271517451Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.271523121Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.271528721Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.271534532Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.271540032Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.271545842Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.271551382Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.537798586Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_internal_dns_eaab5753-bf35-45e9-8e6d-201b35af6027"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.537858218Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_internal_dns_2da10309-7a29-4b33-90f3-4d4ffabdaad7"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.537865848Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_internal_dns_f4d810c4-7125-4f8d-90af-608014da1d8f"} {"msg":"Writing ledger to /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.537890588Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.538192996Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.538426362Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"edd40f22-bd18-4140-8e54-0f94d421fdaf","remote_addr":"[fd00:1122:3344:101::1]:44532","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":9067450,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.540029393Z","hostname":"sock","pid":1555,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:549"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.586385331Z","hostname":"sock","pid":1555,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:549"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.632985884Z","hostname":"sock","pid":1555,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:549"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.6769009Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:585"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.67691091Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.708148424Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.709913159Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.709919349Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: f4d810c4-7125-4f8d-90af-608014da1d8f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: eaab5753-bf35-45e9-8e6d-201b35af6027, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 73b5765f-e4d3-4379-b5ae-021bdffb805d, kind: Service { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe }, name: Name(\"ntp-271fef89-d32d-4388-88e8-5ee58f9d7dfe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 19])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.737721405Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.737815127Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:41737"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.738051344Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.738064274Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.75836849Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.758411451Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.778080221Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.778115231Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.79618098Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.796229932Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.796391836Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.796537439Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.807492168Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.839015038Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.839706686Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.839716856Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.882536404Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 214, 19]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 }, vnic: \"vopte0\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.882600616Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"ntp"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.883120179Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.883130359Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.883394666Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.884644688Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_ntp_271fef89-d32d-4388-88e8-5ee58f9d7dfe","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.907213711Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_ntp_271fef89-d32d-4388-88e8-5ee58f9d7dfe","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:48.924988803Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_ntp_271fef89-d32d-4388-88e8-5ee58f9d7dfe:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:49.548436672Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:49.548522504Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_271fef89-d32d-4388-88e8-5ee58f9d7dfe","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_ntp_271fef89-d32d-4388-88e8-5ee58f9d7dfe\") failed: Property not found. retry in 41.170335ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:49.68418165Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_271fef89-d32d-4388-88e8-5ee58f9d7dfe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_271fef89-d32d-4388-88e8-5ee58f9d7dfe\") failed: Property not found. retry in 88.947433ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:49.732962709Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_271fef89-d32d-4388-88e8-5ee58f9d7dfe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_271fef89-d32d-4388-88e8-5ee58f9d7dfe\") failed: Property not found. retry in 251.322667ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:49.842938393Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_271fef89-d32d-4388-88e8-5ee58f9d7dfe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_271fef89-d32d-4388-88e8-5ee58f9d7dfe\") failed: Property not found. retry in 400.555297ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:50.105624156Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_271fef89-d32d-4388-88e8-5ee58f9d7dfe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_271fef89-d32d-4388-88e8-5ee58f9d7dfe\") failed: Property not found. retry in 916.297854ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:50.516169076Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_271fef89-d32d-4388-88e8-5ee58f9d7dfe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_271fef89-d32d-4388-88e8-5ee58f9d7dfe\") failed: Property not found. retry in 543.827743ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:51.441665948Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_271fef89-d32d-4388-88e8-5ee58f9d7dfe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:51.509083671Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:41888"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:51.509286186Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"b812ff25-d947-4f55-91fb-ba67af9b1de5","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:41888","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":96,"response_code":"200"} {"msg":"wait for service Some(\"oxz_ntp_271fef89-d32d-4388-88e8-5ee58f9d7dfe\") failed: Property not found. retry in 927.018957ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:51.994612845Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_271fef89-d32d-4388-88e8-5ee58f9d7dfe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/8b6fc4f9-f2be-4116-a97c-1830a2193641): error trying to connect: dns error: no record found for Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:52.727345891Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"wait for service Some(\"oxz_ntp_271fef89-d32d-4388-88e8-5ee58f9d7dfe\") failed: Property not found. retry in 1.25838414s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:52.930668776Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_271fef89-d32d-4388-88e8-5ee58f9d7dfe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_271fef89-d32d-4388-88e8-5ee58f9d7dfe\") failed: Property not found. retry in 1.104477996s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:54.198370301Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_271fef89-d32d-4388-88e8-5ee58f9d7dfe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:55.322867979Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_ntp_271fef89-d32d-4388-88e8-5ee58f9d7dfe"} {"msg":"Writing ledger to /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:55.322956541Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:55.323377852Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:55.323730791Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"e460e459-4b81-416a-bdac-5029a0aa3f1f","remote_addr":"[fd00:1122:3344:101::1]:41737","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":6585832,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:55.325718431Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:626"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:55.354172264Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:55.354345949Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:47628"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_271fef89-d32d-4388-88e8-5ee58f9d7dfe': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:55.363401519Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3633"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:55.363608984Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"52f9fe42-d9c6-470b-b1b2-c4be46ea87b7","remote_addr":"[fd00:1122:3344:101::1]:47628","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":9164,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:55.366306222Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:55.525187259Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:55.525315632Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:57575"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_271fef89-d32d-4388-88e8-5ee58f9d7dfe': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:59.534005202Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3633"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:59.534114455Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"d78d972c-a03b-4554-89ea-1cd78cefb76e","remote_addr":"[fd00:1122:3344:101::1]:57575","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4008691,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:59.537293935Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:59.683779637Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:59.68389453Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:51541"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:59.693187106Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"4a8f0270-4be8-4640-966d-e194baed983d","remote_addr":"[fd00:1122:3344:101::1]:51541","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":9128,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:59.695220457Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:59.695226868Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:59.843569416Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:59.843690689Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:47968"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:59.851546719Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"ec5b9567-eb3c-4f81-a421-351a907521d3","remote_addr":"[fd00:1122:3344:101::1]:47968","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":7743,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:23:59.853534559Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:23:59.85354234Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:00.075807276Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:00.075905828Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:45259"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:00.084043855Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"0f434886-4cd0-4306-af23-33ba001efe56","remote_addr":"[fd00:1122:3344:101::1]:45259","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8017,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:00.085404569Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:00.085410749Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:00.23657704Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:00.236703233Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:38379"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:00.244894661Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"9a8b5a16-974f-45b1-91c6-e5b07829b489","remote_addr":"[fd00:1122:3344:101::1]:38379","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8092,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:00.247881627Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:00.247888397Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:00.433537033Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:00.433623496Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:54903"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:00.44166541Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"fd4356bd-1e51-4bbc-8f2f-fea6bbd29f8b","remote_addr":"[fd00:1122:3344:101::1]:54903","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":7922,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:00.443279771Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:00.443284161Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:00.815028235Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:00.815154429Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:55295"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:00.823041659Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"268b14ee-eda0-44f0-974c-8a51df3d7723","remote_addr":"[fd00:1122:3344:101::1]:55295","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":7765,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:00.825924282Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:00.825930752Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:01.358779459Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:01.358872381Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:64875"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:01.367360157Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"82963624-66b3-43ea-aa4a-3ec41e2f47f6","remote_addr":"[fd00:1122:3344:101::1]:64875","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8403,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:01.368778053Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:01.368792424Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:01.85872594Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:01.858846374Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:43075"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:01.867087133Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"c67807e0-495a-469c-90ff-b0f2ef9f1e5c","remote_addr":"[fd00:1122:3344:101::1]:43075","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8141,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:01.869101514Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:01.869107804Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:02.594540823Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:02.594644946Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:38443"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:02.603433839Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"397a4dac-4c48-42ca-a9a3-2d8563da87fb","remote_addr":"[fd00:1122:3344:101::1]:38443","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8634,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:02.604963458Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:02.604967618Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:03.274759724Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:03.274874687Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:48943"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:03.28365648Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"ce454a18-be47-417c-9713-9cad498e814b","remote_addr":"[fd00:1122:3344:101::1]:48943","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8632,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:03.285792204Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:03.285799634Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:03.964704782Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:03.964834365Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:56409"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:03.973080565Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"c5e26ad6-e532-47d1-af8d-89b6a49cbb5b","remote_addr":"[fd00:1122:3344:101::1]:56409","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8116,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:03.974986013Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:03.974992493Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:04.106751982Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"344.96894ms"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:04.238765976Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:04.238874979Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:45007"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:04.247159389Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"30f3d3e5-4806-4132-877a-711ba8ed9236","remote_addr":"[fd00:1122:3344:101::1]:45007","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8169,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:04.250548726Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:04.250557446Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:05.07536717Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:05.075532334Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:40598"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:05.083966138Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"a0e4d893-5137-4e39-bf1a-77bbb6f2e3ef","remote_addr":"[fd00:1122:3344:101::1]:40598","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8388,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:05.085926778Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:05.085931138Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:06.482886097Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:06.48299363Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:55340"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:06.503698806Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"8704b1a8-c3be-4c6d-b851-548e7423b198","remote_addr":"[fd00:1122:3344:101::1]:55340","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":20569,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 1163513799, ip_addr: 69.89.207.199, stratum: 2, ref_time: 1718918646.1789086, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:06.50659071Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:06.50660716Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1224"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f4d810c4-7125-4f8d-90af-608014da1d8f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: eaab5753-bf35-45e9-8e6d-201b35af6027, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 73b5765f-e4d3-4379-b5ae-021bdffb805d, kind: Service { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe }, name: Name(\"ntp-271fef89-d32d-4388-88e8-5ee58f9d7dfe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 19])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } } } }, OmicronZoneConfig { id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } } } }, OmicronZoneConfig { id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External } } } }, OmicronZoneConfig { id: fb5e251e-d41a-4d5a-85b5-97693d026532, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:06.53650573Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:06.536639493Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:56944"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:06.536861809Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:06.536870859Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:06.555533023Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:06.555575414Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:06.575099619Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:06.57513436Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:06.59281956Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:06.5928546Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:06.648624527Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:06.648672539Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:06.70110952Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:06.701144881Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:06.751499871Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:06.751536872Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:06.803241765Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:06.803271926Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:06.854003315Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:06.854041436Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:06.854272782Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:06.854400805Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:06.88725351Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:06.887297371Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:06.887307021Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:06.887313561Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:06.887321691Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:06.887432384Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"705584ab-0fb3-40e9-8039-52d43f60e299","remote_addr":"[fd00:1122:3344:101::1]:56944","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\", TimeNotSynchronized), (\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\", TimeNotSynchronized), (\"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58\", TimeNotSynchronized), (\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\", TimeNotSynchronized), (\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\", TimeNotSynchronized)]","latency_us":350716,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:06.88768276Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"91.065189ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"705584ab-0fb3-40e9-8039-52d43f60e299\", \"content-length\": \"124\", \"date\": \"Thu, 20 Jun 2024 21:24:06 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"705584ab-0fb3-40e9-8039-52d43f60e299\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f4d810c4-7125-4f8d-90af-608014da1d8f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: eaab5753-bf35-45e9-8e6d-201b35af6027, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 73b5765f-e4d3-4379-b5ae-021bdffb805d, kind: Service { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe }, name: Name(\"ntp-271fef89-d32d-4388-88e8-5ee58f9d7dfe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 19])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } } } }, OmicronZoneConfig { id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } } } }, OmicronZoneConfig { id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External } } } }, OmicronZoneConfig { id: fb5e251e-d41a-4d5a-85b5-97693d026532, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:06.979852902Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:06.980822977Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:06.980828807Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:06.998921606Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:06.998959747Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.016509143Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.016538304Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.037505417Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.037538007Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.056165451Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.056197932Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.074568628Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.074600649Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.092207176Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.092244997Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.110130862Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.110159672Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.129438372Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.129467523Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.129629187Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.12972952Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:07.162526763Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:07.162572084Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:07.162580844Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:07.162589335Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:07.162597665Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.162671447Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"583fbb32-4dcc-4f15-8c39-a31b385b2853","remote_addr":"[fd00:1122:3344:101::1]:56944","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\", TimeNotSynchronized), (\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\", TimeNotSynchronized), (\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\", TimeNotSynchronized), (\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\", TimeNotSynchronized), (\"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58\", TimeNotSynchronized)]","latency_us":182587,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:07.162834041Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"159.613ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"583fbb32-4dcc-4f15-8c39-a31b385b2853\", \"content-length\": \"124\", \"date\": \"Thu, 20 Jun 2024 21:24:06 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"583fbb32-4dcc-4f15-8c39-a31b385b2853\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f4d810c4-7125-4f8d-90af-608014da1d8f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: eaab5753-bf35-45e9-8e6d-201b35af6027, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 73b5765f-e4d3-4379-b5ae-021bdffb805d, kind: Service { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe }, name: Name(\"ntp-271fef89-d32d-4388-88e8-5ee58f9d7dfe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 19])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } } } }, OmicronZoneConfig { id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } } } }, OmicronZoneConfig { id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External } } } }, OmicronZoneConfig { id: fb5e251e-d41a-4d5a-85b5-97693d026532, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.323977235Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.324338264Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.324355644Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.343297826Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.343336356Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.360998455Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.361032446Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.379136476Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.379171617Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.39740408Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.397433311Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.415757046Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.415784847Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.433533628Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.433566109Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.451460013Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.451489944Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.470361534Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.470419285Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.470596649Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.470706832Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:07.502437668Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:07.50247727Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:07.50248621Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:07.5024944Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:07.50250271Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.502603103Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"1c887914-f000-4660-8442-658daa2c602c","remote_addr":"[fd00:1122:3344:101::1]:56944","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\", TimeNotSynchronized), (\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\", TimeNotSynchronized), (\"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58\", TimeNotSynchronized), (\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\", TimeNotSynchronized), (\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\", TimeNotSynchronized)]","latency_us":178456,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:07.502761847Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"161.859561ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1c887914-f000-4660-8442-658daa2c602c\", \"content-length\": \"124\", \"date\": \"Thu, 20 Jun 2024 21:24:06 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1c887914-f000-4660-8442-658daa2c602c\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f4d810c4-7125-4f8d-90af-608014da1d8f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: eaab5753-bf35-45e9-8e6d-201b35af6027, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 73b5765f-e4d3-4379-b5ae-021bdffb805d, kind: Service { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe }, name: Name(\"ntp-271fef89-d32d-4388-88e8-5ee58f9d7dfe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 19])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } } } }, OmicronZoneConfig { id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } } } }, OmicronZoneConfig { id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External } } } }, OmicronZoneConfig { id: fb5e251e-d41a-4d5a-85b5-97693d026532, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.666140257Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.666501866Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.666516397Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.685862278Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.685903939Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.703763583Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.703803024Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.72135989Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.721402361Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.739607914Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.739648805Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.757115728Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.757159329Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.775552637Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.775589568Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.793154824Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.793196525Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.811066219Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.811120831Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.811330956Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.811459299Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:07.844140779Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:07.84418412Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:07.844192481Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:07.844201031Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:07.844215521Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.844303143Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"a8329a75-55ad-4e17-a765-0fee92f0ca1b","remote_addr":"[fd00:1122:3344:101::1]:56944","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58\", TimeNotSynchronized), (\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\", TimeNotSynchronized), (\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\", TimeNotSynchronized), (\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\", TimeNotSynchronized), (\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\", TimeNotSynchronized)]","latency_us":177994,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:07.844471158Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"132.842538ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a8329a75-55ad-4e17-a765-0fee92f0ca1b\", \"content-length\": \"124\", \"date\": \"Thu, 20 Jun 2024 21:24:07 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a8329a75-55ad-4e17-a765-0fee92f0ca1b\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f4d810c4-7125-4f8d-90af-608014da1d8f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: eaab5753-bf35-45e9-8e6d-201b35af6027, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 73b5765f-e4d3-4379-b5ae-021bdffb805d, kind: Service { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe }, name: Name(\"ntp-271fef89-d32d-4388-88e8-5ee58f9d7dfe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 19])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } } } }, OmicronZoneConfig { id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } } } }, OmicronZoneConfig { id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External } } } }, OmicronZoneConfig { id: fb5e251e-d41a-4d5a-85b5-97693d026532, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.978867602Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.979205461Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.979222831Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.997631749Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:07.99766524Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.015482072Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.015520433Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.034007503Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.034047554Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.051465206Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.051497537Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.069533056Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.069561266Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.087969174Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.088008975Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.106562016Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.106608577Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.124543503Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.124589364Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.124774259Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.124878491Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:08.15668053Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:08.156735991Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:08.156745861Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:08.156755391Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:08.156765241Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.156882835Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"2f820c7e-70cb-44d0-846b-8cf8f0918cd9","remote_addr":"[fd00:1122:3344:101::1]:56944","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\", TimeNotSynchronized), (\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\", TimeNotSynchronized), (\"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58\", TimeNotSynchronized), (\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\", TimeNotSynchronized), (\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\", TimeNotSynchronized)]","latency_us":177844,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:08.157068959Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"173.371086ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2f820c7e-70cb-44d0-846b-8cf8f0918cd9\", \"content-length\": \"124\", \"date\": \"Thu, 20 Jun 2024 21:24:07 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2f820c7e-70cb-44d0-846b-8cf8f0918cd9\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f4d810c4-7125-4f8d-90af-608014da1d8f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: eaab5753-bf35-45e9-8e6d-201b35af6027, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 73b5765f-e4d3-4379-b5ae-021bdffb805d, kind: Service { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe }, name: Name(\"ntp-271fef89-d32d-4388-88e8-5ee58f9d7dfe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 19])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } } } }, OmicronZoneConfig { id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } } } }, OmicronZoneConfig { id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External } } } }, OmicronZoneConfig { id: fb5e251e-d41a-4d5a-85b5-97693d026532, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.332221949Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.332601919Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.33263903Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.3503699Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.350413091Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.368199203Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.368236764Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.385219805Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.385263817Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.403533351Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.403569322Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.420894482Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.420938443Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.439347611Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.439381432Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.456698701Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.456733442Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.474629577Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.474690538Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.474879673Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.474999826Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:08.506345853Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:08.506385454Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:08.506409104Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:08.506416064Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:08.506421355Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.506494277Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"31ed1d6a-a0f6-4ec3-95bd-052ba6e1c44c","remote_addr":"[fd00:1122:3344:101::1]:56944","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\", TimeNotSynchronized), (\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\", TimeNotSynchronized), (\"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58\", TimeNotSynchronized), (\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\", TimeNotSynchronized), (\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\", TimeNotSynchronized)]","latency_us":174102,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:08.50664707Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"335.932061ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"31ed1d6a-a0f6-4ec3-95bd-052ba6e1c44c\", \"content-length\": \"124\", \"date\": \"Thu, 20 Jun 2024 21:24:07 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"31ed1d6a-a0f6-4ec3-95bd-052ba6e1c44c\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f4d810c4-7125-4f8d-90af-608014da1d8f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: eaab5753-bf35-45e9-8e6d-201b35af6027, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 73b5765f-e4d3-4379-b5ae-021bdffb805d, kind: Service { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe }, name: Name(\"ntp-271fef89-d32d-4388-88e8-5ee58f9d7dfe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 19])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } } } }, OmicronZoneConfig { id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } } } }, OmicronZoneConfig { id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External } } } }, OmicronZoneConfig { id: fb5e251e-d41a-4d5a-85b5-97693d026532, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.844028081Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.8443522Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.84436193Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.863165418Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.863211719Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.881588575Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.881636397Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.899440849Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.89948055Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.91799163Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.918030521Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.935744931Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.935786692Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.954014006Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.954061987Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.971403137Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.971447778Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.990164514Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.990220625Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.99040046Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:08.990514943Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:09.021750096Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:09.021796857Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:09.021805088Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:09.021813098Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:09.021823078Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:09.02190906Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"416cbae2-0cbf-42b4-9d95-92867e6ba83e","remote_addr":"[fd00:1122:3344:101::1]:56944","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\", TimeNotSynchronized), (\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\", TimeNotSynchronized), (\"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58\", TimeNotSynchronized), (\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\", TimeNotSynchronized), (\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\", TimeNotSynchronized)]","latency_us":177719,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:09.022076755Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"401.957026ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"416cbae2-0cbf-42b4-9d95-92867e6ba83e\", \"content-length\": \"124\", \"date\": \"Thu, 20 Jun 2024 21:24:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"416cbae2-0cbf-42b4-9d95-92867e6ba83e\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f4d810c4-7125-4f8d-90af-608014da1d8f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: eaab5753-bf35-45e9-8e6d-201b35af6027, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 73b5765f-e4d3-4379-b5ae-021bdffb805d, kind: Service { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe }, name: Name(\"ntp-271fef89-d32d-4388-88e8-5ee58f9d7dfe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 19])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } } } }, OmicronZoneConfig { id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } } } }, OmicronZoneConfig { id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External } } } }, OmicronZoneConfig { id: fb5e251e-d41a-4d5a-85b5-97693d026532, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:09.424767035Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:09.425098414Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:09.425105324Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:09.443797518Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:09.44384375Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:09.463296874Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:09.463334285Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:09.481183788Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:09.481222669Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:09.499300179Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:09.49935147Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:09.517577353Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:09.517614624Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:09.535548069Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:09.535592081Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:09.55367651Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:09.553718161Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:09.571552544Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:09.571614896Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:09.571768149Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:09.571890313Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:09.604143411Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:09.604191552Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:09.604199032Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:09.604207353Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:09.604215873Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:09.604321266Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"841f506f-c3be-469e-be58-117af60da3db","remote_addr":"[fd00:1122:3344:101::1]:56944","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\", TimeNotSynchronized), (\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\", TimeNotSynchronized), (\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\", TimeNotSynchronized), (\"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58\", TimeNotSynchronized), (\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\", TimeNotSynchronized)]","latency_us":179350,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:09.60448407Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"389.293848ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"841f506f-c3be-469e-be58-117af60da3db\", \"content-length\": \"124\", \"date\": \"Thu, 20 Jun 2024 21:24:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"841f506f-c3be-469e-be58-117af60da3db\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f4d810c4-7125-4f8d-90af-608014da1d8f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: eaab5753-bf35-45e9-8e6d-201b35af6027, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 73b5765f-e4d3-4379-b5ae-021bdffb805d, kind: Service { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe }, name: Name(\"ntp-271fef89-d32d-4388-88e8-5ee58f9d7dfe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 19])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } } } }, OmicronZoneConfig { id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } } } }, OmicronZoneConfig { id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External } } } }, OmicronZoneConfig { id: fb5e251e-d41a-4d5a-85b5-97693d026532, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:09.994908349Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:09.996027367Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:09.996048067Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:10.014784354Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:10.014837195Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:10.033860978Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:10.033905969Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:10.052258426Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:10.052300367Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:10.07053152Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:10.070562571Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:10.088764573Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:10.088806814Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:10.107085199Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:10.107118749Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:10.125503266Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:10.125544508Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:10.143640737Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:10.143705729Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:10.143875983Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:10.143985076Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:10.175533787Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:10.175572928Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:10.175581529Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:10.175589859Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:10.175596599Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:10.175667721Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"81f12811-4ea7-4a79-91a3-c1a7ad0f757a","remote_addr":"[fd00:1122:3344:101::1]:56944","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\", TimeNotSynchronized), (\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\", TimeNotSynchronized), (\"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58\", TimeNotSynchronized), (\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\", TimeNotSynchronized), (\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\", TimeNotSynchronized)]","latency_us":180612,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:10.175815585Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"324.050608ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"81f12811-4ea7-4a79-91a3-c1a7ad0f757a\", \"content-length\": \"124\", \"date\": \"Thu, 20 Jun 2024 21:24:10 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"81f12811-4ea7-4a79-91a3-c1a7ad0f757a\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f4d810c4-7125-4f8d-90af-608014da1d8f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: eaab5753-bf35-45e9-8e6d-201b35af6027, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 73b5765f-e4d3-4379-b5ae-021bdffb805d, kind: Service { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe }, name: Name(\"ntp-271fef89-d32d-4388-88e8-5ee58f9d7dfe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 19])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } } } }, OmicronZoneConfig { id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } } } }, OmicronZoneConfig { id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External } } } }, OmicronZoneConfig { id: fb5e251e-d41a-4d5a-85b5-97693d026532, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:10.500996216Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:10.501318564Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:10.501331864Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:10.520266665Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:10.520311937Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:10.53776645Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:10.537805741Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:10.555315576Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:10.555366137Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:10.573394985Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:10.573428736Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:10.590599682Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:10.590639313Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:10.608908727Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:10.608946078Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:10.627039598Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:10.627076459Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:10.644504461Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:10.644543102Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:10.644700686Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:10.644799289Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:10.676454423Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:10.676508925Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:10.676520475Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:10.676529865Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:10.676539395Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:10.676609427Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"3fb60961-24f1-4d21-8da5-c3a456d2701e","remote_addr":"[fd00:1122:3344:101::1]:56944","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\", TimeNotSynchronized), (\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\", TimeNotSynchronized), (\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\", TimeNotSynchronized), (\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\", TimeNotSynchronized), (\"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58\", TimeNotSynchronized)]","latency_us":175462,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:10.676751901Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"532.37924ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3fb60961-24f1-4d21-8da5-c3a456d2701e\", \"content-length\": \"124\", \"date\": \"Thu, 20 Jun 2024 21:24:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3fb60961-24f1-4d21-8da5-c3a456d2701e\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f4d810c4-7125-4f8d-90af-608014da1d8f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: eaab5753-bf35-45e9-8e6d-201b35af6027, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 73b5765f-e4d3-4379-b5ae-021bdffb805d, kind: Service { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe }, name: Name(\"ntp-271fef89-d32d-4388-88e8-5ee58f9d7dfe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 19])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } } } }, OmicronZoneConfig { id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } } } }, OmicronZoneConfig { id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External } } } }, OmicronZoneConfig { id: fb5e251e-d41a-4d5a-85b5-97693d026532, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:11.210203653Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:11.210605743Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:11.210612683Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:11.228957829Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:11.22899362Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:11.247635674Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:11.247676395Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:11.265024066Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:11.265072597Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:11.283032973Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:11.283065224Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:11.301591835Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:11.301633946Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:11.319355656Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:11.319393167Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:11.33761483Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:11.337650981Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:11.357210938Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:11.357262639Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:11.357424433Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:11.357531886Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:11.390224257Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:11.390261827Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:11.390270598Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:11.390289468Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:11.390298078Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:11.390399661Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"2e05fedc-c86c-41b4-81e5-2e8de17016eb","remote_addr":"[fd00:1122:3344:101::1]:56944","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\", TimeNotSynchronized), (\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\", TimeNotSynchronized), (\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\", TimeNotSynchronized), (\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\", TimeNotSynchronized), (\"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58\", TimeNotSynchronized)]","latency_us":179985,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:11.390590796Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"665.143009ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2e05fedc-c86c-41b4-81e5-2e8de17016eb\", \"content-length\": \"124\", \"date\": \"Thu, 20 Jun 2024 21:24:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2e05fedc-c86c-41b4-81e5-2e8de17016eb\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f4d810c4-7125-4f8d-90af-608014da1d8f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: eaab5753-bf35-45e9-8e6d-201b35af6027, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 73b5765f-e4d3-4379-b5ae-021bdffb805d, kind: Service { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe }, name: Name(\"ntp-271fef89-d32d-4388-88e8-5ee58f9d7dfe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 19])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } } } }, OmicronZoneConfig { id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } } } }, OmicronZoneConfig { id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External } } } }, OmicronZoneConfig { id: fb5e251e-d41a-4d5a-85b5-97693d026532, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:12.056916944Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:12.057279503Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:12.057292763Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:12.07567706Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:12.075711511Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:12.094391716Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:12.094431627Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:12.111840809Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:12.11187385Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:12.130898243Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:12.130935594Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:12.149547077Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:12.149581778Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:12.168091508Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:12.168136059Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:12.185660285Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:12.185691475Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:12.204032581Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:12.204079553Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:12.204287858Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:12.204415491Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:12.236977228Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:12.237028319Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:12.23703857Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:12.23704677Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:12.23705641Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:12.237157823Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"0148ca33-dfb5-4e5a-afb2-312057dfe9ff","remote_addr":"[fd00:1122:3344:101::1]:56944","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\", TimeNotSynchronized), (\"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58\", TimeNotSynchronized), (\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\", TimeNotSynchronized), (\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\", TimeNotSynchronized), (\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\", TimeNotSynchronized)]","latency_us":180069,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:12.237367558Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"539.936368ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0148ca33-dfb5-4e5a-afb2-312057dfe9ff\", \"content-length\": \"124\", \"date\": \"Thu, 20 Jun 2024 21:24:12 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0148ca33-dfb5-4e5a-afb2-312057dfe9ff\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f4d810c4-7125-4f8d-90af-608014da1d8f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: eaab5753-bf35-45e9-8e6d-201b35af6027, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 73b5765f-e4d3-4379-b5ae-021bdffb805d, kind: Service { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe }, name: Name(\"ntp-271fef89-d32d-4388-88e8-5ee58f9d7dfe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 19])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } } } }, OmicronZoneConfig { id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } } } }, OmicronZoneConfig { id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External } } } }, OmicronZoneConfig { id: fb5e251e-d41a-4d5a-85b5-97693d026532, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:12.778819933Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:12.779184302Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:12.779198543Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:12.798427461Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:12.798473182Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:12.816769357Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:12.816816248Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:12.834749714Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:12.834795515Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:12.853450459Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:12.85349969Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:12.872076352Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:12.872123073Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:12.889967657Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:12.890010648Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:12.90822545Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:12.908272372Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:12.926563236Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:12.926626578Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:12.926777922Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:12.926873824Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:12.959300728Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:12.959358109Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:12.95936661Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:12.95937354Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:12.95938169Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:12.959476112Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"904c7d14-7929-485b-91b3-3841dcf57648","remote_addr":"[fd00:1122:3344:101::1]:56944","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\", TimeNotSynchronized), (\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\", TimeNotSynchronized), (\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\", TimeNotSynchronized), (\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\", TimeNotSynchronized), (\"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58\", TimeNotSynchronized)]","latency_us":180486,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:12.959657607Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"744.20276ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"904c7d14-7929-485b-91b3-3841dcf57648\", \"content-length\": \"124\", \"date\": \"Thu, 20 Jun 2024 21:24:12 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"904c7d14-7929-485b-91b3-3841dcf57648\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f4d810c4-7125-4f8d-90af-608014da1d8f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: eaab5753-bf35-45e9-8e6d-201b35af6027, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 73b5765f-e4d3-4379-b5ae-021bdffb805d, kind: Service { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe }, name: Name(\"ntp-271fef89-d32d-4388-88e8-5ee58f9d7dfe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 19])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } } } }, OmicronZoneConfig { id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } } } }, OmicronZoneConfig { id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External } } } }, OmicronZoneConfig { id: fb5e251e-d41a-4d5a-85b5-97693d026532, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:13.705128635Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:13.705428413Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:13.705466664Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:13.724449766Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:13.724501818Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:13.742657819Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:13.74269841Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:13.760888162Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:13.760932693Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:13.778875909Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:13.77890927Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:13.797324298Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:13.797361149Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:13.815073399Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:13.81512829Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:13.83324098Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:13.833275661Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:13.851145815Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:13.851194226Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:13.851402572Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:13.851530635Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:13.883924588Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:13.883977439Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:13.883983649Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:13.88399313Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:13.88400021Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:13.884070112Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"6b288083-dcb5-45f9-9346-0707939eda15","remote_addr":"[fd00:1122:3344:101::1]:56944","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\", TimeNotSynchronized), (\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\", TimeNotSynchronized), (\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\", TimeNotSynchronized), (\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\", TimeNotSynchronized), (\"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58\", TimeNotSynchronized)]","latency_us":178806,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:13.884268186Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"540.723944ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6b288083-dcb5-45f9-9346-0707939eda15\", \"content-length\": \"124\", \"date\": \"Thu, 20 Jun 2024 21:24:13 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6b288083-dcb5-45f9-9346-0707939eda15\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f4d810c4-7125-4f8d-90af-608014da1d8f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: eaab5753-bf35-45e9-8e6d-201b35af6027, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 73b5765f-e4d3-4379-b5ae-021bdffb805d, kind: Service { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe }, name: Name(\"ntp-271fef89-d32d-4388-88e8-5ee58f9d7dfe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 19])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } } } }, OmicronZoneConfig { id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } } } }, OmicronZoneConfig { id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External } } } }, OmicronZoneConfig { id: fb5e251e-d41a-4d5a-85b5-97693d026532, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:14.426262216Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:14.4264161Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:14.42642837Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:14.445566937Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:14.445622598Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:14.463574414Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:14.463629385Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:14.481445948Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:14.481495239Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:14.500101152Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:14.500143903Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:14.518392567Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:14.518430108Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:14.536082156Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:14.536125067Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:14.553525939Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:14.55357021Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:14.572134231Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:14.572177702Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:14.572389727Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:14.572516251Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:14.604038131Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:14.604095113Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:14.604104813Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:14.604114453Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:14.604123984Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:14.604205326Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"f002f753-4ac1-4bdc-98ac-c99fe435a64e","remote_addr":"[fd00:1122:3344:101::1]:56944","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\", TimeNotSynchronized), (\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\", TimeNotSynchronized), (\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\", TimeNotSynchronized), (\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\", TimeNotSynchronized), (\"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58\", TimeNotSynchronized)]","latency_us":177876,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:14.604399211Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"1.923509896s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f002f753-4ac1-4bdc-98ac-c99fe435a64e\", \"content-length\": \"124\", \"date\": \"Thu, 20 Jun 2024 21:24:14 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f002f753-4ac1-4bdc-98ac-c99fe435a64e\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f4d810c4-7125-4f8d-90af-608014da1d8f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: eaab5753-bf35-45e9-8e6d-201b35af6027, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 73b5765f-e4d3-4379-b5ae-021bdffb805d, kind: Service { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe }, name: Name(\"ntp-271fef89-d32d-4388-88e8-5ee58f9d7dfe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 19])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } } } }, OmicronZoneConfig { id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } } } }, OmicronZoneConfig { id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External } } } }, OmicronZoneConfig { id: fb5e251e-d41a-4d5a-85b5-97693d026532, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:16.52919661Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:16.529816176Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:16.529840797Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:16.549105766Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:16.549165127Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:16.567690478Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:16.567722909Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:16.586388813Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:16.586430104Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:16.604694248Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:16.604723789Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:16.623203178Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:16.623244279Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:16.641687208Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:16.641730009Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:16.659148121Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:16.659187772Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:16.678188065Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:16.678247547Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:16.678521084Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:16.678692278Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:16.710695911Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:16.710746912Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:16.710757792Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:16.710766613Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:16.710792033Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:16.710890706Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"555b1900-0670-4905-bbc9-a1cc17f6d121","remote_addr":"[fd00:1122:3344:101::1]:56944","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\", TimeNotSynchronized), (\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\", TimeNotSynchronized), (\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\", TimeNotSynchronized), (\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\", TimeNotSynchronized), (\"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58\", TimeNotSynchronized)]","latency_us":181356,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:16.71107462Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"1.865815144s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"555b1900-0670-4905-bbc9-a1cc17f6d121\", \"content-length\": \"124\", \"date\": \"Thu, 20 Jun 2024 21:24:16 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"555b1900-0670-4905-bbc9-a1cc17f6d121\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f4d810c4-7125-4f8d-90af-608014da1d8f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: eaab5753-bf35-45e9-8e6d-201b35af6027, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 73b5765f-e4d3-4379-b5ae-021bdffb805d, kind: Service { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe }, name: Name(\"ntp-271fef89-d32d-4388-88e8-5ee58f9d7dfe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 19])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } } } }, OmicronZoneConfig { id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } } } }, OmicronZoneConfig { id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External } } } }, OmicronZoneConfig { id: fb5e251e-d41a-4d5a-85b5-97693d026532, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:18.577746193Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:18.578284116Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:18.578354748Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:18.597876114Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:18.597906135Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:18.617149884Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:18.617180694Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:18.635194332Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:18.635225333Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:18.653824965Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:18.653861596Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:18.672557981Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:18.672592662Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:18.69021467Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:18.690254231Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:18.708887094Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:18.708917215Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:18.72841078Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:18.728463331Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:18.728684707Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:18.728819051Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:18.761647285Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:18.761700806Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:18.761708906Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:18.761718586Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:18.761726087Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:18.761881231Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"da9afeed-96c2-4ac0-9036-f87e6a19611d","remote_addr":"[fd00:1122:3344:101::1]:56944","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\", TimeNotSynchronized), (\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\", TimeNotSynchronized), (\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\", TimeNotSynchronized), (\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\", TimeNotSynchronized), (\"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58\", TimeNotSynchronized)]","latency_us":183845,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:18.762093976Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"2.511345863s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"da9afeed-96c2-4ac0-9036-f87e6a19611d\", \"content-length\": \"124\", \"date\": \"Thu, 20 Jun 2024 21:24:18 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"da9afeed-96c2-4ac0-9036-f87e6a19611d\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:19.57077735Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"552.149531ms"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:21.146698926Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:35519"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:21.146769238Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"1ef5f9be-7aad-4c7a-963d-fa15654d9d4b","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:35519","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":45,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f4d810c4-7125-4f8d-90af-608014da1d8f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: eaab5753-bf35-45e9-8e6d-201b35af6027, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 73b5765f-e4d3-4379-b5ae-021bdffb805d, kind: Service { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe }, name: Name(\"ntp-271fef89-d32d-4388-88e8-5ee58f9d7dfe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 19])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } } } }, OmicronZoneConfig { id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } } } }, OmicronZoneConfig { id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External } } } }, OmicronZoneConfig { id: fb5e251e-d41a-4d5a-85b5-97693d026532, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:21.27438555Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:21.274730269Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:21.274743139Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:21.295230949Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:21.295275091Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:21.315282329Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:21.31532526Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:21.335442151Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:21.335517203Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:21.353875559Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:21.353934441Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:21.372732498Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:21.372775229Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:21.390945661Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:21.391004172Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:21.408953128Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:21.40899746Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:21.426604227Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:21.426658378Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:21.426894944Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:21.427025578Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:21.45939396Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:21.459464112Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:21.459476772Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:21.459486762Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:21.459498192Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:21.459597705Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"c907553d-8bc1-4f05-bbdf-15fdf0ff2e59","remote_addr":"[fd00:1122:3344:101::1]:56944","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\", TimeNotSynchronized), (\"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58\", TimeNotSynchronized), (\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\", TimeNotSynchronized), (\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\", TimeNotSynchronized), (\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\", TimeNotSynchronized)]","latency_us":185060,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:21.459813311Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"2.156443782s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c907553d-8bc1-4f05-bbdf-15fdf0ff2e59\", \"content-length\": \"124\", \"date\": \"Thu, 20 Jun 2024 21:24:21 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c907553d-8bc1-4f05-bbdf-15fdf0ff2e59\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f4d810c4-7125-4f8d-90af-608014da1d8f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: eaab5753-bf35-45e9-8e6d-201b35af6027, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 73b5765f-e4d3-4379-b5ae-021bdffb805d, kind: Service { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe }, name: Name(\"ntp-271fef89-d32d-4388-88e8-5ee58f9d7dfe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 19])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } } } }, OmicronZoneConfig { id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } } } }, OmicronZoneConfig { id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External } } } }, OmicronZoneConfig { id: fb5e251e-d41a-4d5a-85b5-97693d026532, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:23.617539337Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:23.617889406Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:23.617907437Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:23.637123135Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:23.637171506Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:23.656970579Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:23.657027141Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:23.677028569Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:23.67706718Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:23.696198526Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:23.696245987Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:23.715365123Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:23.715428294Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:23.733834472Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:23.733887373Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:23.752684321Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:23.752726152Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:23.77036675Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:23.770428852Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:23.770596776Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:23.770707489Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:23.803369549Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:23.80342846Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:23.80343797Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:23.803459591Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:23.803469501Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:23.803558543Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"8fe42bb7-ab17-4b28-b1e1-690ffeac031c","remote_addr":"[fd00:1122:3344:101::1]:56944","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\", TimeNotSynchronized), (\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\", TimeNotSynchronized), (\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\", TimeNotSynchronized), (\"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58\", TimeNotSynchronized), (\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\", TimeNotSynchronized)]","latency_us":185852,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:23.804258261Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"2.745326153s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8fe42bb7-ab17-4b28-b1e1-690ffeac031c\", \"content-length\": \"124\", \"date\": \"Thu, 20 Jun 2024 21:24:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8fe42bb7-ab17-4b28-b1e1-690ffeac031c\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f4d810c4-7125-4f8d-90af-608014da1d8f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: eaab5753-bf35-45e9-8e6d-201b35af6027, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 73b5765f-e4d3-4379-b5ae-021bdffb805d, kind: Service { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe }, name: Name(\"ntp-271fef89-d32d-4388-88e8-5ee58f9d7dfe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 19])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } } } }, OmicronZoneConfig { id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } } } }, OmicronZoneConfig { id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External } } } }, OmicronZoneConfig { id: fb5e251e-d41a-4d5a-85b5-97693d026532, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:26.55053962Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:26.550934899Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:26.55095443Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:26.570780014Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:26.570823595Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:26.590437863Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:26.590478554Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:26.608406949Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:26.60844771Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:26.62613474Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:26.626188671Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:26.644824924Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:26.644861536Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:26.662450832Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:26.662489223Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:26.680199453Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:26.680255955Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:26.698613351Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:26.698653002Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:26.698821526Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:26.698940799Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:26.729902326Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:26.729949817Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:26.729958047Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:26.729968018Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:26.729976548Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:26.730089871Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"234243af-6432-4e39-920b-e8497883552f","remote_addr":"[fd00:1122:3344:101::1]:56944","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\", TimeNotSynchronized), (\"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58\", TimeNotSynchronized), (\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\", TimeNotSynchronized), (\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\", TimeNotSynchronized), (\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\", TimeNotSynchronized)]","latency_us":179368,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:26.730273995Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"4.199719734s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"234243af-6432-4e39-920b-e8497883552f\", \"content-length\": \"124\", \"date\": \"Thu, 20 Jun 2024 21:24:26 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"234243af-6432-4e39-920b-e8497883552f\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f4d810c4-7125-4f8d-90af-608014da1d8f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: eaab5753-bf35-45e9-8e6d-201b35af6027, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 73b5765f-e4d3-4379-b5ae-021bdffb805d, kind: Service { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe }, name: Name(\"ntp-271fef89-d32d-4388-88e8-5ee58f9d7dfe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 19])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } } } }, OmicronZoneConfig { id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } } } }, OmicronZoneConfig { id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External } } } }, OmicronZoneConfig { id: fb5e251e-d41a-4d5a-85b5-97693d026532, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:30.930613524Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:30.931011564Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:30.931030085Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:30.950757936Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:30.950819177Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:30.969902762Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:30.969945053Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:30.987834318Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:30.987882329Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:31.0056369Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:31.005686821Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:31.023601656Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:31.023651288Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:31.041201804Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:31.041251905Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:31.05836092Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:31.05839341Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:31.076318216Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:31.076353666Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:31.076549771Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:31.076664114Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:31.108028171Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:31.108105403Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:31.108115604Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:31.108120924Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:31.108125814Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:31.108222466Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"ed72e836-d990-4a20-a460-e11dd476e998","remote_addr":"[fd00:1122:3344:101::1]:56944","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\", TimeNotSynchronized), (\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\", TimeNotSynchronized), (\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\", TimeNotSynchronized), (\"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58\", TimeNotSynchronized), (\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\", TimeNotSynchronized)]","latency_us":177416,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:31.108445532Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"5.223216263s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ed72e836-d990-4a20-a460-e11dd476e998\", \"content-length\": \"124\", \"date\": \"Thu, 20 Jun 2024 21:24:30 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ed72e836-d990-4a20-a460-e11dd476e998\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:35.881418533Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"738.403609ms"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f4d810c4-7125-4f8d-90af-608014da1d8f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: eaab5753-bf35-45e9-8e6d-201b35af6027, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 73b5765f-e4d3-4379-b5ae-021bdffb805d, kind: Service { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe }, name: Name(\"ntp-271fef89-d32d-4388-88e8-5ee58f9d7dfe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 19])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } } } }, OmicronZoneConfig { id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } } } }, OmicronZoneConfig { id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External } } } }, OmicronZoneConfig { id: fb5e251e-d41a-4d5a-85b5-97693d026532, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:37.054830616Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:37.055147609Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:37.055163265Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:37.076811686Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:37.076872957Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:37.095226335Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:37.095260796Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:37.113902671Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:37.113937792Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:37.131812977Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:37.131846078Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:37.149554499Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:37.14959433Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:37.168126492Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:37.168174523Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:37.185662159Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:37.18570344Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:37.203456842Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:37.203503223Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:37.203752211Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:37.203886683Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:37.235443937Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:37.235485458Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:37.235494348Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:37.235501658Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:37.235510528Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:37.235624491Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"e1a23a13-89b5-491c-9265-043ec1f037d6","remote_addr":"[fd00:1122:3344:101::1]:56944","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\", TimeNotSynchronized), (\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\", TimeNotSynchronized), (\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\", TimeNotSynchronized), (\"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58\", TimeNotSynchronized), (\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\", TimeNotSynchronized)]","latency_us":181148,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:37.235832387Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"5.566553094s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e1a23a13-89b5-491c-9265-043ec1f037d6\", \"content-length\": \"124\", \"date\": \"Thu, 20 Jun 2024 21:24:37 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e1a23a13-89b5-491c-9265-043ec1f037d6\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f4d810c4-7125-4f8d-90af-608014da1d8f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: eaab5753-bf35-45e9-8e6d-201b35af6027, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 73b5765f-e4d3-4379-b5ae-021bdffb805d, kind: Service { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe }, name: Name(\"ntp-271fef89-d32d-4388-88e8-5ee58f9d7dfe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 19])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } } } }, OmicronZoneConfig { id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } } } }, OmicronZoneConfig { id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External } } } }, OmicronZoneConfig { id: fb5e251e-d41a-4d5a-85b5-97693d026532, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:42.788801741Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:42.789245222Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:42.789254743Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:42.809868018Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:42.809931279Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:42.829005935Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:42.829050116Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:42.846912271Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:42.846963483Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:42.864620443Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:42.864663553Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:42.882459007Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:42.882499448Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:42.899621844Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:42.899666495Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:42.917999122Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:42.918041113Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:42.935369395Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:42.935422556Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:42.935630681Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:42.935762175Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:42.966790745Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:42.966853717Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:42.966863787Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:42.966873157Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:42.966884497Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:42.9669775Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"78413b97-e6b7-4218-b928-94d8f768ec27","remote_addr":"[fd00:1122:3344:101::1]:56944","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\", TimeNotSynchronized), (\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\", TimeNotSynchronized), (\"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58\", TimeNotSynchronized), (\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\", TimeNotSynchronized), (\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\", TimeNotSynchronized)]","latency_us":178432,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:42.967176385Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"5.753395309s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"78413b97-e6b7-4218-b928-94d8f768ec27\", \"content-length\": \"124\", \"date\": \"Thu, 20 Jun 2024 21:24:42 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"78413b97-e6b7-4218-b928-94d8f768ec27\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f4d810c4-7125-4f8d-90af-608014da1d8f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: eaab5753-bf35-45e9-8e6d-201b35af6027, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 73b5765f-e4d3-4379-b5ae-021bdffb805d, kind: Service { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe }, name: Name(\"ntp-271fef89-d32d-4388-88e8-5ee58f9d7dfe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 19])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } } } }, OmicronZoneConfig { id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } } } }, OmicronZoneConfig { id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External } } } }, OmicronZoneConfig { id: fb5e251e-d41a-4d5a-85b5-97693d026532, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:48.706230539Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:48.706702022Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:48.706730922Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:48.729895052Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:48.729944664Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:48.75216711Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:48.752210561Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:48.771339208Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:48.771386189Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:48.790077226Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:48.790125577Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:48.808358651Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:48.808406302Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:48.825781625Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:48.825842706Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:48.842897161Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:48.842947642Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:48.860537941Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:48.860593692Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:48.860779647Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:48.860887709Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:48.891657733Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:48.891695344Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:48.891711104Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:48.891720185Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:48.891727385Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:48.891862948Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"0014dd1b-90a3-42ad-88bd-53553c3d9ede","remote_addr":"[fd00:1122:3344:101::1]:56944","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\", TimeNotSynchronized), (\"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58\", TimeNotSynchronized), (\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\", TimeNotSynchronized), (\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\", TimeNotSynchronized), (\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\", TimeNotSynchronized)]","latency_us":185944,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:48.892081624Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"6.092789156s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0014dd1b-90a3-42ad-88bd-53553c3d9ede\", \"content-length\": \"124\", \"date\": \"Thu, 20 Jun 2024 21:24:48 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0014dd1b-90a3-42ad-88bd-53553c3d9ede\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:51.611502768Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"2.757055751s"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:51.862856231Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:48187"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:51.862980145Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"397def37-28f7-429f-9360-0358addeec5b","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:48187","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":55,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f4d810c4-7125-4f8d-90af-608014da1d8f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: eaab5753-bf35-45e9-8e6d-201b35af6027, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 73b5765f-e4d3-4379-b5ae-021bdffb805d, kind: Service { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe }, name: Name(\"ntp-271fef89-d32d-4388-88e8-5ee58f9d7dfe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 19])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } } } }, OmicronZoneConfig { id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } } } }, OmicronZoneConfig { id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External } } } }, OmicronZoneConfig { id: fb5e251e-d41a-4d5a-85b5-97693d026532, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:54.969230212Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:54.969566961Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:54.969582291Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:54.990715879Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:54.990759001Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:55.009417911Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:55.009467047Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:55.028775959Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:55.02881806Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:55.046735176Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:55.046776928Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:55.064808257Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:55.064857378Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:55.08261059Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:55.082650871Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:55.101911092Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:55.101947373Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:55.119567382Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:55.119620643Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:55.119773827Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:55.11988343Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:55.151399693Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:55.151451244Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:55.151459854Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:55.151474724Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:55.151482535Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:24:55.151574127Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"b860605a-36cc-47e0-b600-e1cc6b725082","remote_addr":"[fd00:1122:3344:101::1]:56944","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\", TimeNotSynchronized), (\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\", TimeNotSynchronized), (\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\", TimeNotSynchronized), (\"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58\", TimeNotSynchronized), (\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\", TimeNotSynchronized)]","latency_us":182687,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:24:55.151787552Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"11.752234439s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b860605a-36cc-47e0-b600-e1cc6b725082\", \"content-length\": \"124\", \"date\": \"Thu, 20 Jun 2024 21:24:54 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b860605a-36cc-47e0-b600-e1cc6b725082\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f4d810c4-7125-4f8d-90af-608014da1d8f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: eaab5753-bf35-45e9-8e6d-201b35af6027, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 73b5765f-e4d3-4379-b5ae-021bdffb805d, kind: Service { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe }, name: Name(\"ntp-271fef89-d32d-4388-88e8-5ee58f9d7dfe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 19])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } } } }, OmicronZoneConfig { id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } } } }, OmicronZoneConfig { id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External } } } }, OmicronZoneConfig { id: fb5e251e-d41a-4d5a-85b5-97693d026532, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:06.873244531Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:06.873643211Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:06.873658671Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:06.894617456Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:06.894656326Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:06.913412504Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:06.913454305Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:06.932195783Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:06.932238824Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:06.950661503Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:06.950699594Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:06.968919467Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:06.96896971Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:06.986859224Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:06.986900512Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:07.004577466Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:07.004641948Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:07.024404852Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:07.024466583Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:07.02474017Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:07.024873614Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:25:07.056444028Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:25:07.056488919Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:25:07.056497639Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:25:07.056504099Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:25:07.05651028Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:07.056657533Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"9b99c00e-1e50-4dc8-9a95-115c37850189","remote_addr":"[fd00:1122:3344:101::1]:56944","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\", TimeNotSynchronized), (\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\", TimeNotSynchronized), (\"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58\", TimeNotSynchronized), (\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\", TimeNotSynchronized), (\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\", TimeNotSynchronized)]","latency_us":183723,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:25:07.056853488Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"11.547769846s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9b99c00e-1e50-4dc8-9a95-115c37850189\", \"content-length\": \"124\", \"date\": \"Thu, 20 Jun 2024 21:25:06 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9b99c00e-1e50-4dc8-9a95-115c37850189\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:25:09.354838117Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"3.612282673s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f4d810c4-7125-4f8d-90af-608014da1d8f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: eaab5753-bf35-45e9-8e6d-201b35af6027, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 73b5765f-e4d3-4379-b5ae-021bdffb805d, kind: Service { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe }, name: Name(\"ntp-271fef89-d32d-4388-88e8-5ee58f9d7dfe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 19])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } } } }, OmicronZoneConfig { id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } } } }, OmicronZoneConfig { id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External } } } }, OmicronZoneConfig { id: fb5e251e-d41a-4d5a-85b5-97693d026532, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:18.57390478Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:18.574965667Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:18.574983538Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:18.595540802Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:18.595590683Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:18.615154621Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:18.615213163Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:18.632642696Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:18.632711448Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:18.652967524Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:18.653017666Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:18.671107236Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:18.671146522Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:18.688851688Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:18.68889404Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:18.705931443Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:18.705983075Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:18.724432305Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:18.724471276Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:18.724788744Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:18.724928247Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:25:18.75563687Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:25:18.755686521Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:25:18.755693391Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:25:18.755701521Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:25:18.755708482Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:18.755813794Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"64f7485b-6227-43e5-b9fc-7b1c24638061","remote_addr":"[fd00:1122:3344:101::1]:56944","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\", TimeNotSynchronized), (\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\", TimeNotSynchronized), (\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\", TimeNotSynchronized), (\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\", TimeNotSynchronized), (\"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58\", TimeNotSynchronized)]","latency_us":182273,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:25:18.756001489Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"11.084020995s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"64f7485b-6227-43e5-b9fc-7b1c24638061\", \"content-length\": \"124\", \"date\": \"Thu, 20 Jun 2024 21:25:18 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"64f7485b-6227-43e5-b9fc-7b1c24638061\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:21.812905579Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:39060"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:21.813024272Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"b32ae770-3851-4916-a7aa-b902e57e3af4","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:39060","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":63,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:25:27.95138257Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"11.464478451s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f4d810c4-7125-4f8d-90af-608014da1d8f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: eaab5753-bf35-45e9-8e6d-201b35af6027, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 73b5765f-e4d3-4379-b5ae-021bdffb805d, kind: Service { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe }, name: Name(\"ntp-271fef89-d32d-4388-88e8-5ee58f9d7dfe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 19])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } } } }, OmicronZoneConfig { id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } } } }, OmicronZoneConfig { id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External } } } }, OmicronZoneConfig { id: fb5e251e-d41a-4d5a-85b5-97693d026532, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:29.811114215Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:29.811475954Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:29.811488695Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:29.833538106Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:29.833567537Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:29.851463343Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:29.851505004Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:29.869053741Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:29.869091232Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:29.888412264Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:29.888454035Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:29.908239439Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:29.90827394Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:29.926106364Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:29.926135775Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:29.944129493Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:29.944157134Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:29.961967408Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:29.962005089Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:29.962268275Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:29.962383908Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:25:29.993670425Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:25:29.993718137Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:25:29.993725127Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:25:29.993734087Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:25:29.993740537Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:29.993872681Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"133a58d0-a44b-4513-87d2-71ac6935f1de","remote_addr":"[fd00:1122:3344:101::1]:56944","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\", TimeNotSynchronized), (\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\", TimeNotSynchronized), (\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\", TimeNotSynchronized), (\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\", TimeNotSynchronized), (\"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58\", TimeNotSynchronized)]","latency_us":183097,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:25:29.994063915Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"18.843753177s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"133a58d0-a44b-4513-87d2-71ac6935f1de\", \"content-length\": \"124\", \"date\": \"Thu, 20 Jun 2024 21:25:29 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"133a58d0-a44b-4513-87d2-71ac6935f1de\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f4d810c4-7125-4f8d-90af-608014da1d8f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: eaab5753-bf35-45e9-8e6d-201b35af6027, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 73b5765f-e4d3-4379-b5ae-021bdffb805d, kind: Service { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe }, name: Name(\"ntp-271fef89-d32d-4388-88e8-5ee58f9d7dfe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 19])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } } } }, OmicronZoneConfig { id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } } } }, OmicronZoneConfig { id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External } } } }, OmicronZoneConfig { id: fb5e251e-d41a-4d5a-85b5-97693d026532, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:48.787753279Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:48.788103747Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:48.788113198Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:48.809526783Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:48.809574974Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:48.828554978Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:48.828585069Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:48.847269084Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:48.847301085Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:48.865361716Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:48.865397326Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:48.883910018Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:48.883943359Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:48.902642805Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:48.902685936Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:48.920538771Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:48.920569942Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:48.93893268Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:48.938976971Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:48.939245038Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:48.939368511Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:25:48.971821388Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:25:48.971857718Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:25:48.971865259Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:25:48.971871839Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:25:48.971877129Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:48.971960971Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"bdba0243-cc0f-4c0e-a964-528ce1f30cf3","remote_addr":"[fd00:1122:3344:101::1]:56944","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\", TimeNotSynchronized), (\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\", TimeNotSynchronized), (\"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58\", TimeNotSynchronized), (\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\", TimeNotSynchronized), (\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\", TimeNotSynchronized)]","latency_us":184556,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:25:48.972141306Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"13.055943622s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bdba0243-cc0f-4c0e-a964-528ce1f30cf3\", \"content-length\": \"124\", \"date\": \"Thu, 20 Jun 2024 21:25:48 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"bdba0243-cc0f-4c0e-a964-528ce1f30cf3\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:51.762695592Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:53919"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:25:51.762780585Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"f74f6621-9f14-4e9b-abe2-9a7ba64f6fc7","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:53919","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":46,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:25:54.378432937Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"10.928320849s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f4d810c4-7125-4f8d-90af-608014da1d8f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: eaab5753-bf35-45e9-8e6d-201b35af6027, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 73b5765f-e4d3-4379-b5ae-021bdffb805d, kind: Service { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe }, name: Name(\"ntp-271fef89-d32d-4388-88e8-5ee58f9d7dfe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 19])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } } } }, OmicronZoneConfig { id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } } } }, OmicronZoneConfig { id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External } } } }, OmicronZoneConfig { id: fb5e251e-d41a-4d5a-85b5-97693d026532, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:01.993861584Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:01.994182602Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:01.994193202Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:02.015387492Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:02.015425983Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:02.033143395Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:02.033184916Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:02.049975273Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:02.050010934Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:02.068522896Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:02.068544656Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:02.086089493Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:02.086121124Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:02.103457035Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:02.103501617Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:02.122291635Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:02.122311976Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:02.140250433Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:02.140273024Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:02.140578821Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:02.140709645Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:26:02.172819933Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:26:02.172853723Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:26:02.172860154Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:26:02.172867704Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:26:02.172875294Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:02.172970817Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"c90c1c98-4dbc-4983-8dab-a35fc91e502b","remote_addr":"[fd00:1122:3344:101::1]:56944","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\", TimeNotSynchronized), (\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\", TimeNotSynchronized), (\"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58\", TimeNotSynchronized), (\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\", TimeNotSynchronized), (\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\", TimeNotSynchronized)]","latency_us":179458,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:26:02.17312209Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"22.237405979s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c90c1c98-4dbc-4983-8dab-a35fc91e502b\", \"content-length\": \"124\", \"date\": \"Thu, 20 Jun 2024 21:26:01 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c90c1c98-4dbc-4983-8dab-a35fc91e502b\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:26:20.271784418Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"43.140901774s"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:21.712218413Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:54084"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:21.712327205Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"854283e4-0bec-447f-be5e-c7d0bbdc303a","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:54084","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":45,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f4d810c4-7125-4f8d-90af-608014da1d8f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: eaab5753-bf35-45e9-8e6d-201b35af6027, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 73b5765f-e4d3-4379-b5ae-021bdffb805d, kind: Service { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe }, name: Name(\"ntp-271fef89-d32d-4388-88e8-5ee58f9d7dfe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 19])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } } } }, OmicronZoneConfig { id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } } } }, OmicronZoneConfig { id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External } } } }, OmicronZoneConfig { id: fb5e251e-d41a-4d5a-85b5-97693d026532, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:24.350565771Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:24.350879209Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:24.350890369Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:24.373538766Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:24.373573347Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:24.392361046Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:24.392414707Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:24.411117174Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:24.411147494Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:24.429730508Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:24.429769209Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:24.448235519Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:24.44827799Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:24.466803392Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:24.466839453Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:24.487155551Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:24.487181971Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:24.504987595Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:24.505025966Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:24.505289442Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:24.505410656Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:26:24.537794281Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:26:24.537835592Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:26:24.537843322Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:26:24.537849032Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:26:24.537856082Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:24.537985515Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"4ad6ca93-c35a-42c4-b84e-4da97b5b32b5","remote_addr":"[fd00:1122:3344:101::1]:56944","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58\", TimeNotSynchronized), (\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\", TimeNotSynchronized), (\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\", TimeNotSynchronized), (\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\", TimeNotSynchronized), (\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\", TimeNotSynchronized)]","latency_us":187780,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:26:24.53815668Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"25.257487015s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4ad6ca93-c35a-42c4-b84e-4da97b5b32b5\", \"content-length\": \"124\", \"date\": \"Thu, 20 Jun 2024 21:26:24 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4ad6ca93-c35a-42c4-b84e-4da97b5b32b5\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f4d810c4-7125-4f8d-90af-608014da1d8f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: eaab5753-bf35-45e9-8e6d-201b35af6027, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 73b5765f-e4d3-4379-b5ae-021bdffb805d, kind: Service { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe }, name: Name(\"ntp-271fef89-d32d-4388-88e8-5ee58f9d7dfe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 19])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } } } }, OmicronZoneConfig { id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } } } }, OmicronZoneConfig { id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External } } } }, OmicronZoneConfig { id: fb5e251e-d41a-4d5a-85b5-97693d026532, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:49.728195582Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:49.728650624Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:49.728668094Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:49.751708854Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:49.751767583Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:49.770270284Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:49.770311345Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:49.788530019Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:49.78856125Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:49.806797875Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:49.806830815Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:49.824428764Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:49.824464515Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:49.841480305Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:49.841510759Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:49.860510213Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:49.860548534Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:49.877883676Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:49.877937307Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:49.878268545Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:49.87844397Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:26:49.910948808Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:26:49.910999059Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:26:49.911007409Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:26:49.911016889Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:26:49.91102435Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:49.911180644Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"427ce72f-2ae3-4af0-828d-e573fb23e720","remote_addr":"[fd00:1122:3344:101::1]:56944","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\", TimeNotSynchronized), (\"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58\", TimeNotSynchronized), (\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\", TimeNotSynchronized), (\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\", TimeNotSynchronized), (\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\", TimeNotSynchronized)]","latency_us":183280,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:26:49.91142301Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"34.40239236s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"427ce72f-2ae3-4af0-828d-e573fb23e720\", \"content-length\": \"124\", \"date\": \"Thu, 20 Jun 2024 21:26:49 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"427ce72f-2ae3-4af0-828d-e573fb23e720\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:51.661163333Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:43330"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:26:51.661276966Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"c3ec5ba0-2864-410f-a0c5-1af46973241f","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:43330","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":58,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:18.290377633Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"63.169655626s"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:21.612375558Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:62330"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:21.612431409Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"7504a024-a496-44f6-92cf-a4a36562e871","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:62330","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":41,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f4d810c4-7125-4f8d-90af-608014da1d8f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: eaab5753-bf35-45e9-8e6d-201b35af6027, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 73b5765f-e4d3-4379-b5ae-021bdffb805d, kind: Service { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe }, name: Name(\"ntp-271fef89-d32d-4388-88e8-5ee58f9d7dfe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 19])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } } } }, OmicronZoneConfig { id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } } } }, OmicronZoneConfig { id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External } } } }, OmicronZoneConfig { id: fb5e251e-d41a-4d5a-85b5-97693d026532, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:24.221577235Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:24.221890683Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:24.221902653Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:24.243185335Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:24.243228746Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:24.261015429Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:24.26105462Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:24.278167676Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:24.278213167Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:24.297621902Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:24.29765834Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:24.314953443Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:24.314983914Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:24.334087341Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:24.334121182Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:24.35289628Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:24.352924051Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:24.371843822Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:24.371887324Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:24.37215833Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:24.372272273Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:24.453650866Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:24.453703018Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:24.453710098Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:24.453715618Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:24.453721748Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:24.549926989Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:24.554173717Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:24.558361894Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:24.56292239Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:24.567809465Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:24.572772681Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:24.577578983Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:24.581969946Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:24.586397528Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:24.590937654Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:27.511572623Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:27.511779978Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:27.518222942Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:27.518368146Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:27.554037415Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:27.554157968Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:27.56604761Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:27.566190154Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:27.572207637Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:27.57232547Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\") failed: Property not found. retry in 65.887877ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:27.650793409Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58\") failed: Property not found. retry in 49.343422ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:27.658204808Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\") failed: Property not found. retry in 46.092542ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:27.694498683Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\") failed: Property not found. retry in 26.869447ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:27.703252696Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\") failed: Property not found. retry in 45.256844ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:27.71049124Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58\") failed: Property not found. retry in 92.655842ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:27.739463328Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\") failed: Property not found. retry in 123.777915ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:27.894913908Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\") failed: Property not found. retry in 75.45035ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:28.215541106Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\") failed: Property not found. retry in 139.449586ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:28.60119068Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\") failed: Property not found. retry in 159.786444ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:28.730202686Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\") failed: Property not found. retry in 101.533928ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:29.201807369Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\") failed: Property not found. retry in 168.759172ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:29.506430749Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58\") failed: Property not found. retry in 266.666362ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:30.448929599Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\") failed: Property not found. retry in 228.777464ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:30.544320518Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\") failed: Property not found. retry in 419.390634ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:30.562367478Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\") failed: Property not found. retry in 127.232159ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:30.593982944Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\") failed: Property not found. retry in 385.086863ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:30.613770328Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58\") failed: Property not found. retry in 393.633025ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:30.726458898Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\") failed: Property not found. retry in 284.142656ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:30.735618532Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\") failed: Property not found. retry in 520.00359ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:30.783108061Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\") failed: Property not found. retry in 1.005337938s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:30.989692854Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\") failed: Property not found. retry in 938.432495ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:31.008093923Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\") failed: Property not found. retry in 994.617038ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:31.030576445Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58\") failed: Property not found. retry in 1.09285693s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:31.129855534Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\") failed: Property not found. retry in 823.168764ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:31.312044335Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\") failed: Property not found. retry in 732.676747ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:31.953937856Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\") failed: Property not found. retry in 1.043980277s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:32.000832571Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\") failed: Property not found. retry in 996.211098ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:32.031986505Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\") failed: Property not found. retry in 1.385256826s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:32.141463463Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58\") failed: Property not found. retry in 1.12531609s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:32.227956707Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\") failed: Property not found. retry in 1.111010543s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:32.694202594Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\") failed: Property not found. retry in 1.137481878s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:33.035622331Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\") failed: Property not found. retry in 1.254352165s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:33.050939262Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58\") failed: Property not found. retry in 896.412068ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:33.363721269Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\") failed: Property not found. retry in 1.060988375s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:33.536235364Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\") failed: Property not found. retry in 856.177692ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:33.815102898Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\") failed: Property not found. retry in 1.229015697s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:34.179780932Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58\") failed: Property not found. retry in 859.358727ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:34.26661659Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\") failed: Property not found. retry in 732.601305ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:34.310521968Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\") failed: Property not found. retry in 817.756569ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:34.602750113Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\") failed: Property not found. retry in 894.648842ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:34.676527712Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\") failed: Property not found. retry in 1.039811128s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:35.048272952Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58\") failed: Property not found. retry in 628.677547ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:35.132394795Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\") failed: Property not found. retry in 1.433773094s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:35.413410245Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\") failed: Property not found. retry in 665.348058ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:35.426958469Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\") failed: Property not found. retry in 1.402253025s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:35.577883903Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58\") failed: Property not found. retry in 1.313962084s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:35.76931892Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\") failed: Property not found. retry in 1.32956101s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:36.097345556Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\") failed: Property not found. retry in 690.760332ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:36.105530811Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\") failed: Property not found. retry in 1.192124267s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:36.804223343Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\") failed: Property not found. retry in 1.037216509s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:36.851963588Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\") failed: Property not found. retry in 833.389327ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:36.986057274Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\") failed: Property not found. retry in 753.275959ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:37.434208941Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\") failed: Property not found. retry in 795.081323ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:37.830623212Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\") failed: Property not found. retry in 751.007252ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:37.903398581Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\") failed: Property not found. retry in 741.548101ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:38.005579116Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\") failed: Property not found. retry in 1.336956753s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:38.197372149Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\") failed: Property not found. retry in 822.340211ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:38.635185612Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\") failed: Property not found. retry in 1.318189317s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:38.664061265Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\") failed: Property not found. retry in 1.227092569s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:38.759223163Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\") failed: Property not found. retry in 1.272307041s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:39.470931894Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\") failed: Property not found. retry in 1.415638096s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:39.545260962Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\") failed: Property not found. retry in 1.403317257s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:39.991753505Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\") failed: Property not found. retry in 1.07789123s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:40.000999129Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\") failed: Property not found. retry in 1.468604767s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:40.753528948Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\") failed: Property not found. retry in 1.454328625s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:40.969416932Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\") failed: Property not found. retry in 745.363184ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:41.08741488Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\") failed: Property not found. retry in 683.797791ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:41.40275774Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\") failed: Property not found. retry in 700.523863ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:41.841368952Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\") failed: Property not found. retry in 984.229705ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:42.095315913Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\") failed: Property not found. retry in 812.779636ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:42.230019054Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\") failed: Property not found. retry in 1.096692203s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:42.434387377Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\") failed: Property not found. retry in 1.167716427s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:42.550984428Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\") failed: Property not found. retry in 1.209052775s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:43.053255136Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\") failed: Property not found. retry in 708.51335ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:43.089333942Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\") failed: Property not found. retry in 1.006139734s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:43.539678192Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\") failed: Property not found. retry in 1.327794641s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:43.727903693Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\") failed: Property not found. retry in 1.324811546s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:43.806754986Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\") failed: Property not found. retry in 730.5639ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:44.270927607Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\") failed: Property not found. retry in 998.87067ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:44.55607262Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\") failed: Property not found. retry in 1.090115069s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:45.010305688Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\") failed: Property not found. retry in 1.407171213s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:45.065955842Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\") failed: Property not found. retry in 1.493645922s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:45.14099259Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\") failed: Property not found. retry in 1.091295667s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:45.564779563Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5\") failed: Property not found. retry in 1.061106493s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:46.109862159Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682\") failed: Property not found. retry in 1.186323801s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:46.481974581Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5\") failed: Property not found. retry in 1.383631473s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:46.643367181Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532\") failed: Property not found. retry in 872.21227ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:27:46.664885648Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:48.053889131Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_cockroachdb_d799bd92-dc1c-47a6-aa8b-8551b50f4682"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:48.053952732Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_cockroachdb_fb5e251e-d41a-4d5a-85b5-97693d026532"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:48.053961012Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_cockroachdb_59cbec31-893a-4cf0-97de-5cf02b83f4a5"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:48.053966023Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_cockroachdb_bcd1fe58-64ab-49d6-8128-de04559a0ac5"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:48.053989893Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_cockroachdb_0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58"} {"msg":"Writing ledger to /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:48.053995313Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:48.054440655Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:48.054771893Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"36f5d35e-f39d-4851-98c4-5cce48fcdf98","remote_addr":"[fd00:1122:3344:101::1]:56944","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":23867113,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:48.088873109Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:53760"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::5]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:48.120539984Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3467"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:27:48.315408064Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3485"} {"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:01.84889759Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3508"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:01.849466984Z","hostname":"sock","pid":1555,"uri":"/cockroachdb","method":"POST","req_id":"b38960be-8563-4203-839c-9dce60add469","remote_addr":"[fd00:1122:3344:101::1]:53760","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":13759778,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:01.851625489Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:64259"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:01.852175963Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"5509b03f-fdef-41d2-b5b7-1dfe1ce6dd8c","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:64259","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":360,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: f4d810c4-7125-4f8d-90af-608014da1d8f, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: eaab5753-bf35-45e9-8e6d-201b35af6027, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 73b5765f-e4d3-4379-b5ae-021bdffb805d, kind: Service { id: 271fef89-d32d-4388-88e8-5ee58f9d7dfe }, name: Name(\"ntp-271fef89-d32d-4388-88e8-5ee58f9d7dfe\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 19])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } } } }, OmicronZoneConfig { id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } } } }, OmicronZoneConfig { id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External } } } }, OmicronZoneConfig { id: fb5e251e-d41a-4d5a-85b5-97693d026532, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: e3ef572f-48a3-41b6-9bbd-9c7f7f828b14, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 08ef7d24-0a5e-4f66-b10f-e9a99f4f350e, kind: Service { id: e3ef572f-48a3-41b6-9bbd-9c7f7f828b14 }, name: Name(\"external-dns-e3ef572f-48a3-41b6-9bbd-9c7f7f828b14\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 66])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 50c4b53f-46eb-4906-832f-6fe24fcc1c27, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 14d2a703-40c9-467c-a24d-276aa20f3e98, kind: Service { id: 50c4b53f-46eb-4906-832f-6fe24fcc1c27 }, name: Name(\"external-dns-50c4b53f-46eb-4906-832f-6fe24fcc1c27\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 158])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 4162f617-779a-4702-98c1-16953936e276, kind: Service { id: 4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c }, name: Name(\"nexus-4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 8])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8f64d0bd-6f7c-418c-a532-2ff9143ec947, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: f7672b04-24b2-4e2f-9016-12a5c768bb8d, kind: Service { id: 8f64d0bd-6f7c-418c-a532-2ff9143ec947 }, name: Name(\"nexus-8f64d0bd-6f7c-418c-a532-2ff9143ec947\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 152, 21])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 65b469d2-6eb7-4e46-8276-ad6d27b67bf9, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 342b451a-1b71-482c-ba92-ff847d794986, kind: Service { id: 65b469d2-6eb7-4e46-8276-ad6d27b67bf9 }, name: Name(\"nexus-65b469d2-6eb7-4e46-8276-ad6d27b67bf9\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 36])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 83a45644-f784-4b40-a7e1-ec94b0dc35d4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 507fce68-85b0-4613-9106-4012a97d4676, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } } } }, OmicronZoneConfig { id: 901e3843-977c-41c6-a5fe-a1f4da0212a4, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 5ba6d79e-6aa9-4e02-b1d7-5104861adf44, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 5d333696-ea20-4be9-82d3-02eb3bf0cfe5, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5b03b56c-61f4-4585-ba9d-0b729d34b0f0, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External } } } }, OmicronZoneConfig { id: a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External } } } }, OmicronZoneConfig { id: c8a7a67f-05dc-422f-a62f-c4b571c77275, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External } } } }, OmicronZoneConfig { id: 47ca0972-bbd2-4407-9ef5-4857eff51a30, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External } } } }, OmicronZoneConfig { id: d3965df4-756a-4af5-92ce-b29fc27c0e10, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: a370f79e-dbb0-4bdc-a8c2-5f357850df6e, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c3976b41-1b46-4fb3-86b8-14634aafccad (zpool), kind: External } } } }, OmicronZoneConfig { id: 499298b1-a521-438e-b48f-ef9b349410af, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd582fd3-6899-4123-a5b6-2a268c4ed13b (zpool), kind: External } } } }, OmicronZoneConfig { id: 131a7779-cd75-4691-b98f-177ec1388d36, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a553f0be-01ff-4b59-9e74-27e6ddf33491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 52a493e4-e262-43f5-a647-e8d25520263f, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ff7b7614-aeaa-4dfa-8c5e-e5115ba48aa0 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:01.891172464Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:01.891472981Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:53385"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:01.892653321Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:01.892665072Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f4d810c4-7125-4f8d-90af-608014da1d8f, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:01.919036632Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:01.919068752Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: eaab5753-bf35-45e9-8e6d-201b35af6027, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:01.938460655Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:01.938583808Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2da10309-7a29-4b33-90f3-4d4ffabdaad7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:01.95755142Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:01.957587661Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 59cbec31-893a-4cf0-97de-5cf02b83f4a5, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:01.977343783Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:01.977391444Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bcd1fe58-64ab-49d6-8128-de04559a0ac5, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:01.996477739Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:01.99651272Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ee6a1aa-73af-4c63-9ca4-4986e3a6fe58, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:02.017091873Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:02.017139394Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d799bd92-dc1c-47a6-aa8b-8551b50f4682, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:02.037626184Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:02.037659565Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fb5e251e-d41a-4d5a-85b5-97693d026532, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3ef572f-48a3-41b6-9bbd-9c7f7f828b14, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:02.06028145Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:02.060317151Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e3ef572f-48a3-41b6-9bbd-9c7f7f828b14, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 50c4b53f-46eb-4906-832f-6fe24fcc1c27, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:02.163527843Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:02.163570254Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 50c4b53f-46eb-4906-832f-6fe24fcc1c27, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 507fce68-85b0-4613-9106-4012a97d4676, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:02.25747827Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:02.257518321Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 507fce68-85b0-4613-9106-4012a97d4676, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5b03b56c-61f4-4585-ba9d-0b729d34b0f0, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:02.362006815Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:02.362057067Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5b03b56c-61f4-4585-ba9d-0b729d34b0f0, dataset_name: DatasetName { pool_name: ZpoolName { id: 68941de4-9d96-43c2-b5bb-c76900595397 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:02.47278221Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:02.472825451Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344, dataset_name: DatasetName { pool_name: ZpoolName { id: c28d26f6-634d-4800-9bba-2148c8387fee (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8a7a67f-05dc-422f-a62f-c4b571c77275, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:02.555429359Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:02.55546924Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c8a7a67f-05dc-422f-a62f-c4b571c77275, dataset_name: DatasetName { pool_name: ZpoolName { id: 3dcbe67c-4d44-4c02-82ee-c644a9177203 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 47ca0972-bbd2-4407-9ef5-4857eff51a30, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:02.658617381Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:02.658657972Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 47ca0972-bbd2-4407-9ef5-4857eff51a30, dataset_name: DatasetName { pool_name: ZpoolName { id: ea2c3cdd-ebf6-434c-a220-e4b6e1180947 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d3965df4-756a-4af5-92ce-b29fc27c0e10, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:02.747448488Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:02.747581101Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d3965df4-756a-4af5-92ce-b29fc27c0e10, dataset_name: DatasetName { pool_name: ZpoolName { id: 05b3cfac-441e-4e11-8952-0d7b0cba33c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a370f79e-dbb0-4bdc-a8c2-5f357850df6e, dataset_name: DatasetName { pool_name: ZpoolName { id: c3976b41-1b46-4fb3-86b8-14634aafccad (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:02.878919837Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:02.878959198Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a370f79e-dbb0-4bdc-a8c2-5f357850df6e, dataset_name: DatasetName { pool_name: ZpoolName { id: c3976b41-1b46-4fb3-86b8-14634aafccad (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 499298b1-a521-438e-b48f-ef9b349410af, dataset_name: DatasetName { pool_name: ZpoolName { id: dd582fd3-6899-4123-a5b6-2a268c4ed13b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:02.989592599Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:02.98963009Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 499298b1-a521-438e-b48f-ef9b349410af, dataset_name: DatasetName { pool_name: ZpoolName { id: dd582fd3-6899-4123-a5b6-2a268c4ed13b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 131a7779-cd75-4691-b98f-177ec1388d36, dataset_name: DatasetName { pool_name: ZpoolName { id: a553f0be-01ff-4b59-9e74-27e6ddf33491 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.09316234Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.093203881Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 131a7779-cd75-4691-b98f-177ec1388d36, dataset_name: DatasetName { pool_name: ZpoolName { id: a553f0be-01ff-4b59-9e74-27e6ddf33491 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 52a493e4-e262-43f5-a647-e8d25520263f, dataset_name: DatasetName { pool_name: ZpoolName { id: ff7b7614-aeaa-4dfa-8c5e-e5115ba48aa0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.222978998Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.223018299Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 52a493e4-e262-43f5-a647-e8d25520263f, dataset_name: DatasetName { pool_name: ZpoolName { id: ff7b7614-aeaa-4dfa-8c5e-e5115ba48aa0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.354544771Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.354592302Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.355127366Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.35529901Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.468437104Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.486132844Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.486462162Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.486532374Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.486540654Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.499316549Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.546914738Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.546959489Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.546964499Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.546968699Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.695486633Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.695819371Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.695865192Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.745159014Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.745403851Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.745463512Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.745510623Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.745537244Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.758105833Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.758826442Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.763695885Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.768471657Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.774815488Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.780335518Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.785241663Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.789942772Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.925465215Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.929532718Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.933900129Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.93866013Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.942446476Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.946974391Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.952210934Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.957190691Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.961765767Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_clickhouse_507fce68-85b0-4613-9106-4012a97d4676","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.967271597Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.972265684Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.97763935Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.983956691Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.990764774Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:03.996765256Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.001898857Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_clickhouse_507fce68-85b0-4613-9106-4012a97d4676","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.008130925Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.013360928Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.018440167Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.02405982Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.030534594Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.073967757Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.114122458Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.170333456Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.220783727Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.306743061Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.308535837Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.308554467Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.453743726Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte1\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 160, 36]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte1\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.453815798Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.459085261Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.459100892Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.559827261Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte2\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 152, 21]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte2\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.559891193Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.563172776Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.563198726Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.649270623Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte3\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 255, 158]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte3\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.649332015Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.655986254Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.656000814Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.747005626Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte4\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 247, 66]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte4\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.747071658Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.750847714Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.750860794Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.872782691Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte5\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 246, 8]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte5\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.872846223Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"nexus"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.873597162Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.874477954Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.874499935Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.874515286Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.874524296Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.874532166Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.876238529Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.87625914Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.878819555Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.878834095Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.895502249Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.900002383Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.947684224Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.955205725Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.955299518Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.958734545Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.9589432Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.962788778Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.962825939Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:04.966226535Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:05.155161595Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:05.166333549Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_external_dns_50c4b53f-46eb-4906-832f-6fe24fcc1c27","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:05.175459111Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_nexus_4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:05.186510492Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:05.195830649Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:05.208127271Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:05.218512865Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_50c4b53f-46eb-4906-832f-6fe24fcc1c27","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:05.230235183Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:05.240875513Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:05.251993145Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:07.481461054Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:07.48166926Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:07.662230627Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:07.66238282Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:07.662606546Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:07.680139552Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:07.680357637Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:07.684448341Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:07.684702558Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:07.685115118Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:07.687314934Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:07.687487358Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:07.687676463Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:07.718331322Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:07.903145457Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:07.944007805Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:07.961160781Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:08.084600267Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:08.461713018Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:08.462000865Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30\") failed: Property not found. retry in 69.641114ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:08.831737597Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e\") failed: Property not found. retry in 68.721576ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:08.841266489Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344\") failed: Property not found. retry in 50.880299ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:08.851844568Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36\") failed: Property not found. retry in 66.964385ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:08.863957365Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4\") failed: Property not found. retry in 31.736198ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:08.873850217Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af\") failed: Property not found. retry in 48.88802ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:08.882912247Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4\") failed: Property not found. retry in 68.669653ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:08.89287958Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f\") failed: Property not found. retry in 49.391222ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:08.914014767Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0\") failed: Property not found. retry in 68.252203ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:09.043888927Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10\") failed: Property not found. retry in 26.859344ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:09.100463054Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4\") failed: Property not found. retry in 82.675268ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:12.357859477Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af\") failed: Property not found. retry in 102.91555ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:12.374707525Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4\") failed: Property not found. retry in 99.568721ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:12.387350487Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f\") failed: Property not found. retry in 134.35248ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:12.404887312Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0\") failed: Property not found. retry in 94.907224ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:12.419513864Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30\") failed: Property not found. retry in 147.505631ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:12.431921119Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e\") failed: Property not found. retry in 60.985862ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:12.444632602Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344\") failed: Property not found. retry in 120.343303ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:12.460117315Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10\") failed: Property not found. retry in 143.899756ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:12.472309975Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36\") failed: Property not found. retry in 58.37601ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:12.48629584Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4\") failed: Property not found. retry in 220.616249ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:12.498622734Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af\") failed: Property not found. retry in 191.223153ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:12.510001643Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4\") failed: Property not found. retry in 230.982147ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:12.5216893Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0\") failed: Property not found. retry in 187.622627ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:12.533895019Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e\") failed: Property not found. retry in 177.197014ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:12.545199797Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36\") failed: Property not found. retry in 168.966157ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:12.557792667Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f\") failed: Property not found. retry in 206.581573ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:12.570175851Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30\") failed: Property not found. retry in 249.938365ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:12.595356741Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344\") failed: Property not found. retry in 154.677114ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:12.609388828Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10\") failed: Property not found. retry in 130.771389ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:12.630563185Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af\") failed: Property not found. retry in 465.692639ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:12.714670952Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4\") failed: Property not found. retry in 538.399561ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:12.731970432Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0\") failed: Property not found. retry in 462.677942ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:12.74293091Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e\") failed: Property not found. retry in 395.295028ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:12.757632734Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36\") failed: Property not found. retry in 521.979108ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:12.771704381Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4\") failed: Property not found. retry in 298.623742ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:12.785401559Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344\") failed: Property not found. retry in 596.325261ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:12.79842586Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10\") failed: Property not found. retry in 200.615008ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:12.813476352Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f\") failed: Property not found. retry in 356.508908ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:12.824608425Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30\") failed: Property not found. retry in 460.11901ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:12.858126746Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10\") failed: Property not found. retry in 1.114488513s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:13.027703944Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4\") failed: Property not found. retry in 709.480964ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:13.099090828Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e\") failed: Property not found. retry in 911.1388ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:13.172207156Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af\") failed: Property not found. retry in 874.322098ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:13.19640168Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f\") failed: Property not found. retry in 867.9222ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:13.211842833Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0\") failed: Property not found. retry in 528.432412ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:13.223756065Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4\") failed: Property not found. retry in 809.537645ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:13.287547526Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36\") failed: Property not found. retry in 1.05138134s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:13.315575768Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30\") failed: Property not found. retry in 1.198117644s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:13.338057259Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344\") failed: Property not found. retry in 1.072245932s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:13.412164192Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0\") failed: Property not found. retry in 578.331158ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:13.767744915Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4\") failed: Property not found. retry in 885.811507ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:13.823317386Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af\") failed: Property not found. retry in 1.052698934s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:14.087350664Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f\") failed: Property not found. retry in 1.178439237s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:14.348409426Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e\") failed: Property not found. retry in 1.141889964s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:14.457638071Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4\") failed: Property not found. retry in 912.09194ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:14.569627006Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0\") failed: Property not found. retry in 984.924598ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:14.60558544Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30\") failed: Property not found. retry in 1.477518667s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:14.621936685Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344\") failed: Property not found. retry in 1.438279944s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:14.639408159Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10\") failed: Property not found. retry in 506.613716ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:16.537672745Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36\") failed: Property not found. retry in 951.329141ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:16.588845574Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4\") failed: Property not found. retry in 558.99962ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:16.60953912Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af\") failed: Property not found. retry in 916.22816ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:16.657605771Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4\") failed: Property not found. retry in 709.341562ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:16.686043594Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f\") failed: Property not found. retry in 1.307931312s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:16.709698095Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0\") failed: Property not found. retry in 732.413332ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:16.831314944Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30\") failed: Property not found. retry in 1.386017459s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:17.007662804Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e\") failed: Property not found. retry in 1.437929383s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:17.158705601Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344\") failed: Property not found. retry in 515.315017ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:17.279315295Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10\") failed: Property not found. retry in 837.871686ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:17.375626742Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4\") failed: Property not found. retry in 1.433380925s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:17.459487902Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4\") failed: Property not found. retry in 1.300105649s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:17.48735662Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36\") failed: Property not found. retry in 1.225261087s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:17.553911551Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0\") failed: Property not found. retry in 768.975538ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:17.575961051Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af\") failed: Property not found. retry in 531.453466ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:17.588834308Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:17.779777499Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:17.78059849Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:17.789763853Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:17.790188984Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344\") failed: Property not found. retry in 995.961809ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:17.810404667Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f\") failed: Property not found. retry in 1.426268782s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:18.030654033Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:18.072430914Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:18.072840945Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af\") failed: Property not found. retry in 834.478091ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:18.133832224Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10\") failed: Property not found. retry in 1.201574775s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:18.226058697Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5\") failed: Property not found. retry in 31.782163ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:18.237275942Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44\") failed: Property not found. retry in 47.980968ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:18.248471956Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275\") failed: Property not found. retry in 31.452694ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:18.26081288Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5\") failed: Property not found. retry in 79.819787ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:18.280208093Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275\") failed: Property not found. retry in 147.918777ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:18.303873374Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44\") failed: Property not found. retry in 96.175446ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:18.314109644Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0\") failed: Property not found. retry in 1.490200581s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:18.358177883Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5\") failed: Property not found. retry in 245.085729ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:18.371687507Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30\") failed: Property not found. retry in 784.078093ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:18.407900447Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44\") failed: Property not found. retry in 227.612804ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:18.42140698Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275\") failed: Property not found. retry in 135.279379ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:18.463547951Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e\") failed: Property not found. retry in 1.372458351s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:18.606912773Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275\") failed: Property not found. retry in 479.526462ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:18.618318583Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5\") failed: Property not found. retry in 323.500674ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:18.629099916Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44\") failed: Property not found. retry in 290.332158ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:18.661103119Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:18.713844389Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:18.714173608Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36\") failed: Property not found. retry in 600.426784ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:18.792662522Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4\") failed: Property not found. retry in 876.295907ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:18.804378929Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344\") failed: Property not found. retry in 912.098159ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:18.82015591Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4\") failed: Property not found. retry in 888.09296ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:18.907266403Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14\") failed: Property not found. retry in 35.422285ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:18.93743764Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44\") failed: Property not found. retry in 1.059172082s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:18.962580428Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5\") failed: Property not found. retry in 953.332778ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:18.973227469Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af\") failed: Property not found. retry in 1.300597931s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:18.983935151Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14\") failed: Property not found. retry in 146.430108ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:18.995021882Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275\") failed: Property not found. retry in 1.094215643s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:19.112807745Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14\") failed: Property not found. retry in 238.063412ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:19.158929777Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30\") failed: Property not found. retry in 573.749728ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:19.206150026Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36\") failed: Property not found. retry in 886.842323ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:19.405656395Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14\") failed: Property not found. retry in 413.324718ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:19.417495055Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10\") failed: Property not found. retry in 1.005503661s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:19.440150211Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f\") failed: Property not found. retry in 1.265005719s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:19.469673081Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_clickhouse_507fce68-85b0-4613-9106-4012a97d4676:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:19.549698244Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:19.54991979Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_507fce68-85b0-4613-9106-4012a97d4676","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4\") failed: Property not found. retry in 815.229768ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:19.694661877Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:19.732061327Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:19.732101468Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:19.732254412Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_clickhouse_507fce68-85b0-4613-9106-4012a97d4676\") failed: Property not found. retry in 68.743628ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:19.750249389Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_507fce68-85b0-4613-9106-4012a97d4676","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344\") failed: Property not found. retry in 875.378567ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:19.761825843Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:19.761943556Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30\") failed: Property not found. retry in 1.285542801s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:19.792995535Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4\") failed: Property not found. retry in 644.33029ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:19.806488458Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_507fce68-85b0-4613-9106-4012a97d4676\") failed: Property not found. retry in 59.914298ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:19.830700983Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_507fce68-85b0-4613-9106-4012a97d4676","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14\") failed: Property not found. retry in 933.906867ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:19.844958825Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0\") failed: Property not found. retry in 893.088243ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:19.860283084Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_507fce68-85b0-4613-9106-4012a97d4676\") failed: Property not found. retry in 244.904239ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:19.901623374Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_507fce68-85b0-4613-9106-4012a97d4676","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5\") failed: Property not found. retry in 1.013778139s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:19.939180909Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947\") failed: Property not found. retry in 38.746107ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:19.957046343Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9\") failed: Property not found. retry in 71.808615ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:19.979055072Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e\") failed: Property not found. retry in 1.063169095s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:19.989338833Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947\") failed: Property not found. retry in 122.47634ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:20.007024502Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44\") failed: Property not found. retry in 598.972392ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:20.034429759Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9\") failed: Property not found. retry in 101.939011ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:20.063333313Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947\") failed: Property not found. retry in 175.02459ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:20.140300558Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_507fce68-85b0-4613-9106-4012a97d4676\") failed: Property not found. retry in 245.813589ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:20.16003045Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_507fce68-85b0-4613-9106-4012a97d4676","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9\") failed: Property not found. retry in 108.868793ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:20.175488442Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275\") failed: Property not found. retry in 1.038573363s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:20.219514271Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9\") failed: Property not found. retry in 470.730737ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:20.299556724Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af\") failed: Property not found. retry in 1.244915254s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:20.309459646Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36\") failed: Property not found. retry in 1.46437444s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:20.320078296Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947\") failed: Property not found. retry in 265.779575ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:20.33284882Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_507fce68-85b0-4613-9106-4012a97d4676\") failed: Property not found. retry in 917.685543ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:20.41904165Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_507fce68-85b0-4613-9106-4012a97d4676","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10\") failed: Property not found. retry in 890.464725ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:20.457694592Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4\") failed: Property not found. retry in 1.344400705s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:20.469518322Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4\") failed: Property not found. retry in 873.412875ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:20.52611713Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947\") failed: Property not found. retry in 576.798511ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:20.609861038Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44\") failed: Property not found. retry in 713.588927ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:20.645798731Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344\") failed: Property not found. retry in 1.39966852s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:20.656974804Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f\") failed: Property not found. retry in 1.166181056s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:20.746201231Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0\") failed: Property not found. retry in 1.104928375s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:20.765991944Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9\") failed: Property not found. retry in 480.179848ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:20.781921889Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14\") failed: Property not found. retry in 666.528862ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:20.792568529Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5\") failed: Property not found. retry in 1.409678838s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:20.96348068Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e\") failed: Property not found. retry in 808.404396ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:21.065419Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30\") failed: Property not found. retry in 1.248133266s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:21.089885962Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947\") failed: Property not found. retry in 1.484864395s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:21.197749622Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275\") failed: Property not found. retry in 1.353127528s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:21.267799812Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9\") failed: Property not found. retry in 685.503587ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:21.275707293Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_507fce68-85b0-4613-9106-4012a97d4676\") failed: Property not found. retry in 1.328332039s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:21.344957222Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_507fce68-85b0-4613-9106-4012a97d4676","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10\") failed: Property not found. retry in 1.453299283s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:21.358049554Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44\") failed: Property not found. retry in 1.451117257s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:21.369262119Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4\") failed: Property not found. retry in 980.129117ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:21.413521184Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14\") failed: Property not found. retry in 608.084509ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:21.492715466Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af\") failed: Property not found. retry in 690.015084ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:21.567452404Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36\") failed: Property not found. retry in 1.382709172s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:21.799422798Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4\") failed: Property not found. retry in 1.120210492s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:21.826958027Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0\") failed: Property not found. retry in 502.085185ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:21.931857612Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f\") failed: Property not found. retry in 987.632007ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:21.944261337Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e\") failed: Property not found. retry in 1.079177324s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:21.955894542Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9\") failed: Property not found. retry in 547.123824ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:21.973852689Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344\") failed: Property not found. retry in 866.984295ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:22.074688511Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14\") failed: Property not found. retry in 1.494656235s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:22.11128019Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af\") failed: Property not found. retry in 510.696685ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:22.27149562Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30\") failed: Property not found. retry in 1.447736923s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:22.403119664Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4\") failed: Property not found. retry in 1.165866539s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:22.419320736Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5\") failed: Property not found. retry in 843.03367ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:22.434612694Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0\") failed: Property not found. retry in 1.201019768s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:22.484510592Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9\") failed: Property not found. retry in 1.398942879s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:22.534385639Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275\") failed: Property not found. retry in 860.340402ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:22.636180265Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_507fce68-85b0-4613-9106-4012a97d4676\") failed: Property not found. retry in 1.171119493s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:22.685206541Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_507fce68-85b0-4613-9106-4012a97d4676","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947\") failed: Property not found. retry in 836.977992ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:22.696825356Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af\") failed: Property not found. retry in 679.551788ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:22.792907727Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10\") failed: Property not found. retry in 1.431371164s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:22.823553455Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44\") failed: Property not found. retry in 836.90516ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:22.835281353Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f\") failed: Property not found. retry in 1.010139527s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:22.943677677Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344\") failed: Property not found. retry in 1.159493381s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:22.955366634Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4\") failed: Property not found. retry in 1.380237316s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:22.966870436Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e\") failed: Property not found. retry in 652.562792ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:23.047252988Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36\") failed: Property not found. retry in 1.312488834s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:23.19531536Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5\") failed: Property not found. retry in 924.465647ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:23.290551549Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:23.29725013Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:23.297363662Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af\") failed: Property not found. retry in 1.142307264s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:23.485735048Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275\") failed: Property not found. retry in 1.409213558s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:23.509574444Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947\") failed: Property not found. retry in 1.105618803s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:23.545791094Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c\") failed: Property not found. retry in 64.425104ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:23.555961202Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4\") failed: Property not found. retry in 1.217231824s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:23.599091128Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14\") failed: Property not found. retry in 841.237898ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:23.619175548Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c\") failed: Property not found. retry in 87.120358ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:23.631783609Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44\") failed: Property not found. retry in 1.252942893s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:23.684278452Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0\") failed: Property not found. retry in 897.454297ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:23.697487408Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e\") failed: Property not found. retry in 1.332783467s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:23.712837668Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c\") failed: Property not found. retry in 284.079466ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:23.729794308Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30\") failed: Property not found. retry in 838.81615ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:23.862826398Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_507fce68-85b0-4613-9106-4012a97d4676\") failed: Property not found. retry in 550.39534ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:23.872877703Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_507fce68-85b0-4613-9106-4012a97d4676","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9\") failed: Property not found. retry in 969.496834ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:23.944625976Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f\") failed: Property not found. retry in 1.382398781s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:23.964893811Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c\") failed: Property not found. retry in 433.464919ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:24.028080337Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344\") failed: Property not found. retry in 1.344475695s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:24.127304357Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5\") failed: Property not found. retry in 729.229489ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:24.226314533Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10\") failed: Property not found. retry in 1.230251996s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:24.266703299Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4\") failed: Property not found. retry in 559.245782ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:24.357368792Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_507fce68-85b0-4613-9106-4012a97d4676\") failed: Property not found. retry in 911.58265ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:24.434071401Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_507fce68-85b0-4613-9106-4012a97d4676","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14\") failed: Property not found. retry in 981.869043ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:24.470797394Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c\") failed: Property not found. retry in 905.737577ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:24.479917465Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36\") failed: Property not found. retry in 922.378621ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:24.518510986Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0\") failed: Property not found. retry in 791.568853ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:24.605864525Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af\") failed: Property not found. retry in 582.577719ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:24.640006643Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947\") failed: Property not found. retry in 859.34078ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:24.661026226Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30\") failed: Property not found. retry in 787.922854ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:24.711324534Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4\") failed: Property not found. retry in 1.245664758s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:24.826440599Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9\") failed: Property not found. retry in 637.408508ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:24.923267379Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4\") failed: Property not found. retry in 1.057407856s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:24.930869572Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275\") failed: Property not found. retry in 1.25219369s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:24.938506276Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44\") failed: Property not found. retry in 1.417285273s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:24.946798337Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5\") failed: Property not found. retry in 1.29155291s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:24.969164705Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e\") failed: Property not found. retry in 1.379660179s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:25.059143361Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af\") failed: Property not found. retry in 633.703118ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:25.231984362Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_507fce68-85b0-4613-9106-4012a97d4676\") failed: Property not found. retry in 902.294301ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:25.354691649Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_507fce68-85b0-4613-9106-4012a97d4676","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f\") failed: Property not found. retry in 1.315610667s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:25.363256077Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c\") failed: Property not found. retry in 923.422927ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:25.394440569Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0\") failed: Property not found. retry in 1.336971752s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:25.406546257Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36\") failed: Property not found. retry in 674.911371ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:25.449463417Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14\") failed: Property not found. retry in 606.110499ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:25.461359439Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344\") failed: Property not found. retry in 944.15591ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:25.481610824Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10\") failed: Property not found. retry in 1.218580614s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:25.505572022Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30\") failed: Property not found. retry in 615.725041ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:25.513207396Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947\") failed: Property not found. retry in 695.794153ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:25.529542851Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9\") failed: Property not found. retry in 1.251815487s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:25.568776138Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af\") failed: Property not found. retry in 989.766739ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:25.880018715Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4\") failed: Property not found. retry in 1.302923194s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:25.997728195Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14\") failed: Property not found. retry in 620.324583ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:26.076844065Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4\") failed: Property not found. retry in 963.18905ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:26.085665999Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36\") failed: Property not found. retry in 838.697461ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:26.135718481Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30\") failed: Property not found. retry in 880.208618ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:26.145286504Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275\") failed: Property not found. retry in 610.326973ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:26.199980224Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947\") failed: Property not found. retry in 947.37918ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:26.234949422Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_507fce68-85b0-4613-9106-4012a97d4676\") failed: Property not found. retry in 1.29240806s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:26.266658948Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_507fce68-85b0-4613-9106-4012a97d4676","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5\") failed: Property not found. retry in 1.161149439s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:26.273796469Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c\") failed: Property not found. retry in 644.325715ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:26.327534304Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44\") failed: Property not found. retry in 1.46350688s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:26.371748758Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344\") failed: Property not found. retry in 1.0988655s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:26.43523944Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e\") failed: Property not found. retry in 505.198888ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:26.449133913Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f\") failed: Property not found. retry in 509.14296ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:26.688747861Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14\") failed: Property not found. retry in 565.509863ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:26.706258416Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10\") failed: Property not found. retry in 1.321927589s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:26.734538864Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0\") failed: Property not found. retry in 1.226525607s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:26.75287044Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275\") failed: Property not found. retry in 1.005872954s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:26.818457725Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9\") failed: Property not found. retry in 1.226134289s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:26.829819484Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af\") failed: Property not found. retry in 1.078858615s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:26.879729502Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e\") failed: Property not found. retry in 837.942705ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:26.964113936Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c\") failed: Property not found. retry in 1.381656969s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:26.981154988Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36\") failed: Property not found. retry in 1.116876679s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:26.987783837Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30\") failed: Property not found. retry in 901.83704ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:27.035594722Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4\") failed: Property not found. retry in 1.228984592s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:27.058713679Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947\") failed: Property not found. retry in 1.236291553s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:27.192503728Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f\") failed: Property not found. retry in 1.153928475s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:27.208616117Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14\") failed: Property not found. retry in 976.436849ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:27.282039223Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4\") failed: Property not found. retry in 1.18997579s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:27.30950344Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5\") failed: Property not found. retry in 848.652651ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:27.444442909Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344\") failed: Property not found. retry in 885.306261ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:27.54371262Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_507fce68-85b0-4613-9106-4012a97d4676\") failed: Property not found. retry in 543.371928ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:27.569488225Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_507fce68-85b0-4613-9106-4012a97d4676","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e\") failed: Property not found. retry in 700.775584ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:27.81135551Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275\") failed: Property not found. retry in 549.62567ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:27.833545043Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44\") failed: Property not found. retry in 640.52164ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:27.843414684Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30\") failed: Property not found. retry in 1.386044497s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:27.947371585Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af\") failed: Property not found. retry in 739.536131ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:27.966916112Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0\") failed: Property not found. retry in 546.815333ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:27.987760631Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9\") failed: Property not found. retry in 715.147438ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:28.066184984Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10\") failed: Property not found. retry in 1.314314847s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:28.074745781Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36\") failed: Property not found. retry in 542.835159ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:28.11368904Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_507fce68-85b0-4613-9106-4012a97d4676\") failed: Property not found. retry in 912.426975ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:28.122359171Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_507fce68-85b0-4613-9106-4012a97d4676","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14\") failed: Property not found. retry in 1.345005196s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:28.269004636Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4\") failed: Property not found. retry in 1.017270784s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:28.296735441Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5\") failed: Property not found. retry in 1.253970819s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:28.305317719Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f\") failed: Property not found. retry in 1.08284133s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:28.371693735Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c\") failed: Property not found. retry in 1.141381883s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:28.379789201Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275\") failed: Property not found. retry in 1.205565209s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:28.392103304Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947\") failed: Property not found. retry in 1.407563081s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:28.439631451Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344\") failed: Property not found. retry in 1.102985906s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:28.449301577Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44\") failed: Property not found. retry in 555.044072ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:28.496550717Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4\") failed: Property not found. retry in 947.005585ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:28.511523767Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e\") failed: Property not found. retry in 662.600925ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:28.522912037Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0\") failed: Property not found. retry in 503.43736ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:28.544402013Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36\") failed: Property not found. retry in 1.04464406s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:28.672020835Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af\") failed: Property not found. retry in 1.415599076s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:28.720373873Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9\") failed: Property not found. retry in 1.352008325s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:28.866469494Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_507fce68-85b0-4613-9106-4012a97d4676\") failed: Property not found. retry in 1.44953321s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:29.044966039Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_507fce68-85b0-4613-9106-4012a97d4676","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0\") failed: Property not found. retry in 874.836371ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:29.056892932Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44\") failed: Property not found. retry in 681.657673ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:29.065569222Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e\") failed: Property not found. retry in 1.330761567s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:29.194014535Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4\") failed: Property not found. retry in 950.042999ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:29.323459534Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30\") failed: Property not found. retry in 962.209263ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:29.342216401Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10\") failed: Property not found. retry in 940.108066ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:29.398066349Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f\") failed: Property not found. retry in 817.883837ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:29.463753608Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4\") failed: Property not found. retry in 927.070015ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:29.47171635Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c\") failed: Property not found. retry in 1.21699956s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:29.530163815Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344\") failed: Property not found. retry in 1.39268855s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:29.561480521Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5\") failed: Property not found. retry in 570.019006ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:29.569161226Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275\") failed: Property not found. retry in 783.744977ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:29.606072854Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14\") failed: Property not found. retry in 873.862882ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:29.622717467Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36\") failed: Property not found. retry in 1.160133308s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:29.725029836Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44\") failed: Property not found. retry in 779.713938ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:29.756051664Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947\") failed: Property not found. retry in 866.650797ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:29.855722026Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0\") failed: Property not found. retry in 784.303391ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:29.939577316Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af\") failed: Property not found. retry in 740.451918ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:30.144510083Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5\") failed: Property not found. retry in 1.458987955s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:30.152984178Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9\") failed: Property not found. retry in 507.767841ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:30.228299231Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4\") failed: Property not found. retry in 1.482427301s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:30.282412146Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f\") failed: Property not found. retry in 763.751759ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:30.290230825Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30\") failed: Property not found. retry in 1.112382576s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:30.314103871Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10\") failed: Property not found. retry in 836.935906ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:30.348012293Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275\") failed: Property not found. retry in 1.472813198s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:30.400000163Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4\") failed: Property not found. retry in 573.173007ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:30.408600302Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_507fce68-85b0-4613-9106-4012a97d4676\") failed: Property not found. retry in 1.482511689s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:30.505074273Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_507fce68-85b0-4613-9106-4012a97d4676","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14\") failed: Property not found. retry in 714.74802ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:30.514493402Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e\") failed: Property not found. retry in 669.032969ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:30.53646679Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44\") failed: Property not found. retry in 1.135235541s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:30.546962357Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_50c4b53f-46eb-4906-832f-6fe24fcc1c27:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:30.708311956Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:30.708517051Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_50c4b53f-46eb-4906-832f-6fe24fcc1c27","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947\") failed: Property not found. retry in 1.119238341s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:30.733385583Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0\") failed: Property not found. retry in 1.178307751s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:30.743750766Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9\") failed: Property not found. retry in 1.218323122s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:30.752688073Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c\") failed: Property not found. retry in 878.357067ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:30.761290242Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_50c4b53f-46eb-4906-832f-6fe24fcc1c27\") failed: Property not found. retry in 42.005548ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:30.88837176Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_50c4b53f-46eb-4906-832f-6fe24fcc1c27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af\") failed: Property not found. retry in 1.273940716s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:30.897525793Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36\") failed: Property not found. retry in 993.407573ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:30.90686532Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_50c4b53f-46eb-4906-832f-6fe24fcc1c27\") failed: Property not found. retry in 98.252553ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:30.939245613Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_50c4b53f-46eb-4906-832f-6fe24fcc1c27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344\") failed: Property not found. retry in 962.443452ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:30.980100191Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4\") failed: Property not found. retry in 938.477895ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:31.03794199Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_50c4b53f-46eb-4906-832f-6fe24fcc1c27\") failed: Property not found. retry in 163.898044ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:31.057032065Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_50c4b53f-46eb-4906-832f-6fe24fcc1c27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f\") failed: Property not found. retry in 1.303185125s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:31.091916031Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10\") failed: Property not found. retry in 1.0438836s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:31.195448912Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e\") failed: Property not found. retry in 773.454657ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:31.2166196Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_50c4b53f-46eb-4906-832f-6fe24fcc1c27\") failed: Property not found. retry in 299.254731ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:31.232899653Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_50c4b53f-46eb-4906-832f-6fe24fcc1c27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14\") failed: Property not found. retry in 1.113077133s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:31.240682811Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30\") failed: Property not found. retry in 1.434366781s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:31.436368302Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_50c4b53f-46eb-4906-832f-6fe24fcc1c27\") failed: Property not found. retry in 837.147578ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:31.542869078Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_50c4b53f-46eb-4906-832f-6fe24fcc1c27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5\") failed: Property not found. retry in 1.326502745s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:31.621887705Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c\") failed: Property not found. retry in 910.862674ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:31.648242165Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44\") failed: Property not found. retry in 1.431798534s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:31.691786241Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4\") failed: Property not found. retry in 1.486812374s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:31.773403405Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947\") failed: Property not found. retry in 1.134583732s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:31.861936723Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275\") failed: Property not found. retry in 528.568178ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:31.884941227Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36\") failed: Property not found. retry in 614.223089ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:31.912470677Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0\") failed: Property not found. retry in 977.482819ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:31.93464616Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344\") failed: Property not found. retry in 629.58116ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:31.95549405Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9\") failed: Property not found. retry in 1.245153556s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:31.983229755Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4\") failed: Property not found. retry in 1.487811688s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:31.9921113Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e\") failed: Property not found. retry in 678.343363ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:31.999705143Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_507fce68-85b0-4613-9106-4012a97d4676\") failed: Property not found. retry in 1.103910687s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:32.007129771Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_507fce68-85b0-4613-9106-4012a97d4676","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:32.138514929Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45897"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:32.141205658Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"8d6cff80-92b5-4f6b-a228-a5dcb6e72d78","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45897","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1801,"response_code":"200"} {"msg":"wait for service Some(\"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af\") failed: Property not found. retry in 1.188446593s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:32.18300225Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10\") failed: Property not found. retry in 1.043958273s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:32.250804482Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14\") failed: Property not found. retry in 597.564483ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:32.364707636Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_50c4b53f-46eb-4906-832f-6fe24fcc1c27\") failed: Property not found. retry in 1.003705331s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:32.391746263Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_50c4b53f-46eb-4906-832f-6fe24fcc1c27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f\") failed: Property not found. retry in 981.221315ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:32.4073846Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275\") failed: Property not found. retry in 912.885916ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:32.425742266Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36\") failed: Property not found. retry in 1.31621976s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:32.537480955Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c\") failed: Property not found. retry in 1.479921809s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:32.57074728Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344\") failed: Property not found. retry in 1.196477784s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:32.596535745Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e\") failed: Property not found. retry in 1.177657422s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:32.691254162Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30\") failed: Property not found. retry in 1.143481909s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:32.883229819Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0\") failed: Property not found. retry in 1.301705652s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:32.926532849Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5\") failed: Property not found. retry in 1.110979221s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:32.960368329Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14\") failed: Property not found. retry in 1.296070248s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:32.974302413Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947\") failed: Property not found. retry in 1.053196422s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:33.018891855Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_507fce68-85b0-4613-9106-4012a97d4676\") failed: Property not found. retry in 1.0475944s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:33.128688025Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_507fce68-85b0-4613-9106-4012a97d4676","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44\") failed: Property not found. retry in 1.006792798s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:33.143301976Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9\") failed: Property not found. retry in 590.092996ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:33.244037625Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4\") failed: Property not found. retry in 961.150357ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:33.274023657Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10\") failed: Property not found. retry in 1.408686541s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:33.309630242Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275\") failed: Property not found. retry in 1.041397907s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:33.352356337Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af\") failed: Property not found. retry in 796.774449ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:33.387083599Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f\") failed: Property not found. retry in 576.62277ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:33.402915312Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_50c4b53f-46eb-4906-832f-6fe24fcc1c27\") failed: Property not found. retry in 1.328550933s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:33.415805819Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_50c4b53f-46eb-4906-832f-6fe24fcc1c27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4\") failed: Property not found. retry in 1.365095562s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:33.494755215Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344\") failed: Property not found. retry in 1.215979656s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:33.806543545Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9\") failed: Property not found. retry in 906.883391ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:33.844421838Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36\") failed: Property not found. retry in 858.688517ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:33.862847076Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e\") failed: Property not found. retry in 1.028799006s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:33.882609658Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f\") failed: Property not found. retry in 1.463980441s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:33.993111985Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30\") failed: Property not found. retry in 622.733853ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:34.050542784Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c\") failed: Property not found. retry in 1.469059446s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:34.068400288Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5\") failed: Property not found. retry in 1.105878138s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:34.087735349Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947\") failed: Property not found. retry in 1.355852284s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:34.099665802Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44\") failed: Property not found. retry in 1.193826521s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:34.162730974Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_507fce68-85b0-4613-9106-4012a97d4676\") failed: Property not found. retry in 1.382412061s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:34.188675934Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_507fce68-85b0-4613-9106-4012a97d4676","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af\") failed: Property not found. retry in 837.184506ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:34.202653569Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0\") failed: Property not found. retry in 626.634008ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:34.246680887Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4\") failed: Property not found. retry in 731.862602ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:34.266440079Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14\") failed: Property not found. retry in 1.034045515s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:34.288618313Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275\") failed: Property not found. retry in 956.708911ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:34.413715321Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30\") failed: Property not found. retry in 1.119346729s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:34.686584013Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10\") failed: Property not found. retry in 1.354949181s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:34.729932064Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36\") failed: Property not found. retry in 826.657389ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:34.741291173Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_50c4b53f-46eb-4906-832f-6fe24fcc1c27\") failed: Property not found. retry in 766.051499ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:34.756987412Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_50c4b53f-46eb-4906-832f-6fe24fcc1c27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9\") failed: Property not found. retry in 1.246263306s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:34.767077288Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4\") failed: Property not found. retry in 1.166675347s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:34.87465194Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0\") failed: Property not found. retry in 1.402145021s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:34.888202114Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e\") failed: Property not found. retry in 1.354566542s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:34.926173099Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4\") failed: Property not found. retry in 881.850082ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:35.014041641Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344\") failed: Property not found. retry in 1.113663708s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:35.034934352Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af\") failed: Property not found. retry in 755.107222ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:35.051738869Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5\") failed: Property not found. retry in 910.593626ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:35.20527491Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14\") failed: Property not found. retry in 1.190754163s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:35.338984127Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44\") failed: Property not found. retry in 1.115278871s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:35.367252034Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275\") failed: Property not found. retry in 1.034253592s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:35.382862041Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947\") failed: Property not found. retry in 1.484241666s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:35.466548017Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f\") failed: Property not found. retry in 1.304653961s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:35.47765811Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_50c4b53f-46eb-4906-832f-6fe24fcc1c27\") failed: Property not found. retry in 543.564852ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:35.534802011Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_50c4b53f-46eb-4906-832f-6fe24fcc1c27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c\") failed: Property not found. retry in 881.030058ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:35.547138845Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36\") failed: Property not found. retry in 793.433959ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:35.57885464Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_507fce68-85b0-4613-9106-4012a97d4676\") failed: Property not found. retry in 1.43559883s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:35.588724321Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_507fce68-85b0-4613-9106-4012a97d4676","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30\") failed: Property not found. retry in 638.816305ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:35.81933851Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af\") failed: Property not found. retry in 1.414083371s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:35.831440397Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4\") failed: Property not found. retry in 1.309704547s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:35.907658954Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9\") failed: Property not found. retry in 751.481987ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:36.025502447Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4\") failed: Property not found. retry in 549.395217ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:36.053441297Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_50c4b53f-46eb-4906-832f-6fe24fcc1c27\") failed: Property not found. retry in 859.547704ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:36.088780185Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_50c4b53f-46eb-4906-832f-6fe24fcc1c27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10\") failed: Property not found. retry in 556.903447ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:36.097617919Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5\") failed: Property not found. retry in 640.54574ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:36.12833947Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344\") failed: Property not found. retry in 910.546979ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:36.158512256Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e\") failed: Property not found. retry in 1.185721767s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:36.294681966Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0\") failed: Property not found. retry in 710.82387ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:36.306367963Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36\") failed: Property not found. retry in 556.186458ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:36.38773975Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275\") failed: Property not found. retry in 1.480496329s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:36.431104912Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c\") failed: Property not found. retry in 930.990052ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:36.442082431Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30\") failed: Property not found. retry in 1.016732181s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:36.467766733Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44\") failed: Property not found. retry in 1.458749887s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:36.494167474Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14\") failed: Property not found. retry in 645.274976ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:36.543241781Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:36.587080224Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"147.359833803s"} {"msg":"wait for service Some(\"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4\") failed: Property not found. retry in 1.212782938s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:36.615424174Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10\") failed: Property not found. retry in 1.048164605s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:36.670054542Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5\") failed: Property not found. retry in 525.779292ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:36.782971911Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9\") failed: Property not found. retry in 1.132136949s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:36.795789286Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f\") failed: Property not found. retry in 658.761618ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:36.805903243Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36\") failed: Property not found. retry in 1.077761042s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:36.958969502Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947\") failed: Property not found. retry in 542.431296ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:36.971300245Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_50c4b53f-46eb-4906-832f-6fe24fcc1c27\") failed: Property not found. retry in 1.417603568s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:36.983716041Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_50c4b53f-46eb-4906-832f-6fe24fcc1c27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0\") failed: Property not found. retry in 1.251017104s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:37.033135766Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_507fce68-85b0-4613-9106-4012a97d4676\") failed: Property not found. retry in 1.014345239s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:37.043375966Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_507fce68-85b0-4613-9106-4012a97d4676","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344\") failed: Property not found. retry in 958.386628ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:37.081033393Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14\") failed: Property not found. retry in 1.368556336s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:37.206420358Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4\") failed: Property not found. retry in 698.648006ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:37.230152321Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af\") failed: Property not found. retry in 613.450177ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:37.265952261Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5\") failed: Property not found. retry in 1.073627418s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:37.326665833Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c\") failed: Property not found. retry in 892.276003ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:37.388569146Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f\") failed: Property not found. retry in 998.841607ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:37.477409023Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e\") failed: Property not found. retry in 781.179739ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:37.492686931Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30\") failed: Property not found. retry in 1.332356776s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:37.506844771Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947\") failed: Property not found. retry in 765.239221ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:37.528785788Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10\") failed: Property not found. retry in 723.834836ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:37.73274941Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4\") failed: Property not found. retry in 727.665578ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:37.84648909Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af\") failed: Property not found. retry in 560.627777ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:37.895973147Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275\") failed: Property not found. retry in 1.047002663s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:37.928284237Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9\") failed: Property not found. retry in 512.124828ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:37.9437285Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4\") failed: Property not found. retry in 524.785599ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:37.959595233Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44\") failed: Property not found. retry in 674.181314ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:37.972721366Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36\") failed: Property not found. retry in 853.899098ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:38.047954528Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344\") failed: Property not found. retry in 991.085178ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:38.059322886Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_507fce68-85b0-4613-9106-4012a97d4676\") failed: Property not found. retry in 1.447996328s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:38.070288885Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_507fce68-85b0-4613-9106-4012a97d4676","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e\") failed: Property not found. retry in 1.114923855s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:38.285581585Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0\") failed: Property not found. retry in 559.413526ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:38.294660525Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c\") failed: Property not found. retry in 1.126498668s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:38.303248354Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947\") failed: Property not found. retry in 1.382062767s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:38.311697228Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5\") failed: Property not found. retry in 1.20083518s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:38.40938976Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_50c4b53f-46eb-4906-832f-6fe24fcc1c27\") failed: Property not found. retry in 1.233873868s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:38.417902636Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_50c4b53f-46eb-4906-832f-6fe24fcc1c27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9\") failed: Property not found. retry in 1.275923665s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:38.465393733Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af\") failed: Property not found. retry in 1.072092514s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:38.474130165Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10\") failed: Property not found. retry in 1.171939617s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:38.482575859Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f\") failed: Property not found. retry in 639.181699ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:38.490601253Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4\") failed: Property not found. retry in 754.941773ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:38.498176986Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4\") failed: Property not found. retry in 700.36985ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:38.583646317Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14\") failed: Property not found. retry in 999.260598ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:38.592154293Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44\") failed: Property not found. retry in 691.150396ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:38.656035096Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30\") failed: Property not found. retry in 744.956032ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:38.849825139Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0\") failed: Property not found. retry in 1.239583631s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:38.863720941Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36\") failed: Property not found. retry in 764.853745ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:38.91091242Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275\") failed: Property not found. retry in 1.478267572s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:38.991149769Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344\") failed: Property not found. retry in 595.497324ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:39.061098876Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f\") failed: Property not found. retry in 887.285603ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:39.140591206Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:39.288441842Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"wait for service Some(\"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4\") failed: Property not found. retry in 1.308747496s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:39.296618609Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e\") failed: Property not found. retry in 762.896428ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:39.409933808Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c\") failed: Property not found. retry in 1.285816642s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:39.440394212Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14\") failed: Property not found. retry in 1.262137472s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:39.603479605Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30\") failed: Property not found. retry in 1.22691682s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:39.615976703Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_50c4b53f-46eb-4906-832f-6fe24fcc1c27\") failed: Property not found. retry in 1.030234276s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:39.666164108Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_50c4b53f-46eb-4906-832f-6fe24fcc1c27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344\") failed: Property not found. retry in 825.095173ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:39.679505037Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10\") failed: Property not found. retry in 1.179446233s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:39.691342667Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9\") failed: Property not found. retry in 757.742596ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:28:39.766443055Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:40.959115105Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_nexus_65b469d2-6eb7-4e46-8276-ad6d27b67bf9"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:40.95931366Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_crucible_pantry_901e3843-977c-41c6-a5fe-a1f4da0212a4"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:40.959345951Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_crucible_499298b1-a521-438e-b48f-ef9b349410af"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:40.959356641Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_oximeter_83a45644-f784-4b40-a7e1-ec94b0dc35d4"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:40.959365061Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_nexus_8f64d0bd-6f7c-418c-a532-2ff9143ec947"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:40.959374492Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_external_dns_50c4b53f-46eb-4906-832f-6fe24fcc1c27"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:40.959382802Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_crucible_52a493e4-e262-43f5-a647-e8d25520263f"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:40.959392412Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_crucible_5b03b56c-61f4-4585-ba9d-0b729d34b0f0"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:40.959400522Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_crucible_pantry_5d333696-ea20-4be9-82d3-02eb3bf0cfe5"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:40.959408962Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_crucible_47ca0972-bbd2-4407-9ef5-4857eff51a30"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:40.959423103Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_crucible_c8a7a67f-05dc-422f-a62f-c4b571c77275"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:40.959430363Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_crucible_a370f79e-dbb0-4bdc-a8c2-5f357850df6e"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:40.959437483Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_nexus_4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:40.959444803Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_clickhouse_507fce68-85b0-4613-9106-4012a97d4676"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:40.959451213Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_crucible_pantry_5ba6d79e-6aa9-4e02-b1d7-5104861adf44"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:40.959470444Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_crucible_a1b5f2d2-6c4b-4f16-a7a6-bca680fc9344"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:40.959476544Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_crucible_d3965df4-756a-4af5-92ce-b29fc27c0e10"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:40.959769831Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_external_dns_e3ef572f-48a3-41b6-9bbd-9c7f7f828b14"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:40.959792542Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_crucible_131a7779-cd75-4691-b98f-177ec1388d36"} {"msg":"Writing ledger to /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:40.960706015Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:40.967400145Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:40.98055167Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"1cfa1bba-a56e-4c9f-91f6-0a9cddf428ae","remote_addr":"[fd00:1122:3344:101::1]:53385","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":39087683,"response_code":"204"} {"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:41.018855252Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1244"} {"msg":"Writing ledger to /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:41.019819877Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:41.020142735Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:41.028260571Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:681"} {"msg":"Nexus address: [fd00:1122:3344:101::a]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:41.032476218Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:694"} {"msg":"rack_network_config: RackNetworkConfigV2 {\n bfd: [],\n bgp: [],\n infra_ip_first: 10.151.2.174,\n infra_ip_last: 10.151.2.174,\n ports: [\n PortConfigV2 {\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.2.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n autoneg: false,\n bgp_peers: [],\n port: \"qsfp0\",\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.2.100,\n vlan_id: None,\n },\n ],\n switch: Switch0,\n uplink_port_fec: None,\n uplink_port_speed: Speed40G,\n },\n ],\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:41.083992957Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:825"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/0aedcb47-b77d-4cb1-8181-dc0c6aa83175/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:42.24611119Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/0aedcb47-b77d-4cb1-8181-dc0c6aa83175/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:42.314788765Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/0aedcb47-b77d-4cb1-8181-dc0c6aa83175/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:42.415243027Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/0aedcb47-b77d-4cb1-8181-dc0c6aa83175/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:42.609555574Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/0aedcb47-b77d-4cb1-8181-dc0c6aa83175/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:42.712910399Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/0aedcb47-b77d-4cb1-8181-dc0c6aa83175/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:42.938414528Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/0aedcb47-b77d-4cb1-8181-dc0c6aa83175/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:43.281871524Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/0aedcb47-b77d-4cb1-8181-dc0c6aa83175/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:43.475283977Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/0aedcb47-b77d-4cb1-8181-dc0c6aa83175/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:43.877722511Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/0aedcb47-b77d-4cb1-8181-dc0c6aa83175/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:44.409161713Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/0aedcb47-b77d-4cb1-8181-dc0c6aa83175/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:44.671421075Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/0aedcb47-b77d-4cb1-8181-dc0c6aa83175/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:45.429978115Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/0aedcb47-b77d-4cb1-8181-dc0c6aa83175/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:46.147473503Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/0aedcb47-b77d-4cb1-8181-dc0c6aa83175/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:46.809085391Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/0aedcb47-b77d-4cb1-8181-dc0c6aa83175/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:47.785159858Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/0aedcb47-b77d-4cb1-8181-dc0c6aa83175/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:49.384790916Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/0aedcb47-b77d-4cb1-8181-dc0c6aa83175/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:50.735393377Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"14e42b9c-6f58-4973-80ca-a86d1c3b782c\", \"content-length\": \"173\", \"date\": \"Thu, 20 Jun 2024 21:28:53 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"8b6fc4f9-f2be-4116-a97c-1830a2193641\\\"\", request_id: \"14e42b9c-6f58-4973-80ca-a86d1c3b782c\" }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:53.879870401Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d14ae26f-d460-4d7a-bbea-cceaad2e60cd\", \"content-length\": \"173\", \"date\": \"Thu, 20 Jun 2024 21:28:55 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"8b6fc4f9-f2be-4116-a97c-1830a2193641\\\"\", request_id: \"d14ae26f-d460-4d7a-bbea-cceaad2e60cd\" }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:55.748330207Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"333a1ad2-0e2a-4561-b8b4-101baa662809\", \"content-length\": \"173\", \"date\": \"Thu, 20 Jun 2024 21:28:58 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"8b6fc4f9-f2be-4116-a97c-1830a2193641\\\"\", request_id: \"333a1ad2-0e2a-4561-b8b4-101baa662809\" }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:28:58.318247406Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:02.179493958Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:57983"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:02.179770085Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"e82324c1-dc24-489d-804d-d0b23dcd7fdd","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:57983","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":39,"response_code":"200"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0d2b436a-54b6-46bf-8c5a-205c5feb0b9b\", \"content-length\": \"173\", \"date\": \"Thu, 20 Jun 2024 21:29:03 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"8b6fc4f9-f2be-4116-a97c-1830a2193641\\\"\", request_id: \"0d2b436a-54b6-46bf-8c5a-205c5feb0b9b\" }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:03.080887907Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1d6defd4-1859-434f-bde2-5ada9d362848\", \"content-length\": \"173\", \"date\": \"Thu, 20 Jun 2024 21:29:08 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"8b6fc4f9-f2be-4116-a97c-1830a2193641\\\"\", request_id: \"1d6defd4-1859-434f-bde2-5ada9d362848\" }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:09.087464843Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"12bae0e9-11f4-4455-af11-601c2318fce4\", \"content-length\": \"173\", \"date\": \"Thu, 20 Jun 2024 21:29:11 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"8b6fc4f9-f2be-4116-a97c-1830a2193641\\\"\", request_id: \"12bae0e9-11f4-4455-af11-601c2318fce4\" }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:11.772769272Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d1d115fe-3bf7-4bed-a460-5761c8b8556c\", \"content-length\": \"173\", \"date\": \"Thu, 20 Jun 2024 21:29:16 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"8b6fc4f9-f2be-4116-a97c-1830a2193641\\\"\", request_id: \"d1d115fe-3bf7-4bed-a460-5761c8b8556c\" }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:16.047539811Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fa1b65a1-19ab-4a3e-a416-742101785db4\", \"content-length\": \"173\", \"date\": \"Thu, 20 Jun 2024 21:29:21 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"8b6fc4f9-f2be-4116-a97c-1830a2193641\\\"\", request_id: \"fa1b65a1-19ab-4a3e-a416-742101785db4\" }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:21.835262965Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:22.554090817Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"sled-agent/src/nexus.rs:592","new_generation":"1","old_generation":"1"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:22.673504236Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:37402"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:22.675693236Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:22.762720127Z","hostname":"sock","pid":1555,"uri":"/inventory","method":"GET","req_id":"225a4362-725c-40d2-b4dc-cb80af3cf585","remote_addr":"[fd00:1122:3344:101::c]:37402","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":88623,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:22.766139364Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:22.766215696Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:22.772748672Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:22.77313366Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"GET","req_id":"333ff852-d04b-4388-96a9-5a9888d1fb62","remote_addr":"[fd00:1122:3344:101::c]:37402","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8074,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:31.72620983Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:45842"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:31.729503024Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 08ef7d24-0a5e-4f66-b10f-e9a99f4f350e, kind: Service { id: e3ef572f-48a3-41b6-9bbd-9c7f7f828b14 }, name: Name(\"external-dns-e3ef572f-48a3-41b6-9bbd-9c7f7f828b14\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 66])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 14d2a703-40c9-467c-a24d-276aa20f3e98, kind: Service { id: 50c4b53f-46eb-4906-832f-6fe24fcc1c27 }, name: Name(\"external-dns-50c4b53f-46eb-4906-832f-6fe24fcc1c27\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 158])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 342b451a-1b71-482c-ba92-ff847d794986, kind: Service { id: 65b469d2-6eb7-4e46-8276-ad6d27b67bf9 }, name: Name(\"nexus-65b469d2-6eb7-4e46-8276-ad6d27b67bf9\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 36])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 4162f617-779a-4702-98c1-16953936e276, kind: Service { id: 4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c }, name: Name(\"nexus-4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 8])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f7672b04-24b2-4e2f-9016-12a5c768bb8d, kind: Service { id: 8f64d0bd-6f7c-418c-a532-2ff9143ec947 }, name: Name(\"nexus-8f64d0bd-6f7c-418c-a532-2ff9143ec947\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 152, 21])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:31.729926535Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:31.729956576Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:31.729978326Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:31.729986056Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:31.729991746Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:31.730008867Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:31.730039467Z","hostname":"sock","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"533624ab-531c-455e-864e-70a9a258da2f","remote_addr":"[fd00:1122:3344:101::a]:45842","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":3512,"response_code":"204"} {"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:31.731564366Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:897"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:32.021314387Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:58912"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:32.02143598Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:32.119330027Z","hostname":"sock","pid":1555,"uri":"/inventory","method":"GET","req_id":"fc0ba753-17f0-48ab-9cfe-40d39904b0b7","remote_addr":"[fd00:1122:3344:101::a]:58912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":97933,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:32.119744078Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:32.119791209Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:32.120162438Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:32.120922398Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"GET","req_id":"d0fa914f-34ee-44ce-927a-bfe4c5e7bda2","remote_addr":"[fd00:1122:3344:101::a]:58912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1269,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:32.214758242Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:33142"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:32.214849494Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"5e583e8b-f0ed-4a30-9441-c46f71d51726","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:33142","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":52,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:32.216790353Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:32893"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:32.21704024Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 08ef7d24-0a5e-4f66-b10f-e9a99f4f350e, kind: Service { id: e3ef572f-48a3-41b6-9bbd-9c7f7f828b14 }, name: Name(\"external-dns-e3ef572f-48a3-41b6-9bbd-9c7f7f828b14\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 66])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 14d2a703-40c9-467c-a24d-276aa20f3e98, kind: Service { id: 50c4b53f-46eb-4906-832f-6fe24fcc1c27 }, name: Name(\"external-dns-50c4b53f-46eb-4906-832f-6fe24fcc1c27\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 158])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 342b451a-1b71-482c-ba92-ff847d794986, kind: Service { id: 65b469d2-6eb7-4e46-8276-ad6d27b67bf9 }, name: Name(\"nexus-65b469d2-6eb7-4e46-8276-ad6d27b67bf9\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 36])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 4162f617-779a-4702-98c1-16953936e276, kind: Service { id: 4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c }, name: Name(\"nexus-4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 8])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f7672b04-24b2-4e2f-9016-12a5c768bb8d, kind: Service { id: 8f64d0bd-6f7c-418c-a532-2ff9143ec947 }, name: Name(\"nexus-8f64d0bd-6f7c-418c-a532-2ff9143ec947\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 152, 21])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:32.217108211Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:32.217154802Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:32.217174803Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:32.217197464Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:32.217211524Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:32.217232855Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:32.217251525Z","hostname":"sock","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"9ffe7661-4f05-4a0d-b228-a21ccbf5116c","remote_addr":"[fd00:1122:3344:101::a]:32893","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":370,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:32.240750452Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:62533"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:32.241021019Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 08ef7d24-0a5e-4f66-b10f-e9a99f4f350e, kind: Service { id: e3ef572f-48a3-41b6-9bbd-9c7f7f828b14 }, name: Name(\"external-dns-e3ef572f-48a3-41b6-9bbd-9c7f7f828b14\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 66])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 14d2a703-40c9-467c-a24d-276aa20f3e98, kind: Service { id: 50c4b53f-46eb-4906-832f-6fe24fcc1c27 }, name: Name(\"external-dns-50c4b53f-46eb-4906-832f-6fe24fcc1c27\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 158])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 342b451a-1b71-482c-ba92-ff847d794986, kind: Service { id: 65b469d2-6eb7-4e46-8276-ad6d27b67bf9 }, name: Name(\"nexus-65b469d2-6eb7-4e46-8276-ad6d27b67bf9\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 36])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 4162f617-779a-4702-98c1-16953936e276, kind: Service { id: 4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c }, name: Name(\"nexus-4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 8])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f7672b04-24b2-4e2f-9016-12a5c768bb8d, kind: Service { id: 8f64d0bd-6f7c-418c-a532-2ff9143ec947 }, name: Name(\"nexus-8f64d0bd-6f7c-418c-a532-2ff9143ec947\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 152, 21])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:32.241096841Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:32.241117651Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:32.241129642Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:32.241147922Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:32.241165623Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:32.241202413Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:32.241219444Z","hostname":"sock","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"470f9907-ca5f-4c21-bfcf-95c04c60eeca","remote_addr":"[fd00:1122:3344:101::c]:62533","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":371,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:33.395455986Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:63097"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:33.396241236Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 08ef7d24-0a5e-4f66-b10f-e9a99f4f350e, kind: Service { id: e3ef572f-48a3-41b6-9bbd-9c7f7f828b14 }, name: Name(\"external-dns-e3ef572f-48a3-41b6-9bbd-9c7f7f828b14\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 66])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 14d2a703-40c9-467c-a24d-276aa20f3e98, kind: Service { id: 50c4b53f-46eb-4906-832f-6fe24fcc1c27 }, name: Name(\"external-dns-50c4b53f-46eb-4906-832f-6fe24fcc1c27\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 158])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 342b451a-1b71-482c-ba92-ff847d794986, kind: Service { id: 65b469d2-6eb7-4e46-8276-ad6d27b67bf9 }, name: Name(\"nexus-65b469d2-6eb7-4e46-8276-ad6d27b67bf9\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 36])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 4162f617-779a-4702-98c1-16953936e276, kind: Service { id: 4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c }, name: Name(\"nexus-4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 8])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f7672b04-24b2-4e2f-9016-12a5c768bb8d, kind: Service { id: 8f64d0bd-6f7c-418c-a532-2ff9143ec947 }, name: Name(\"nexus-8f64d0bd-6f7c-418c-a532-2ff9143ec947\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 152, 21])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:33.396296128Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:33.396331509Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:33.396348169Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:33.396363549Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:33.39637229Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:33.39638619Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:33.39640323Z","hostname":"sock","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"0cd5633f-8c1b-48bf-ba58-56ae62898d89","remote_addr":"[fd00:1122:3344:101::b]:63097","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":892,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:51.491363483Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:42874"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:51.492628495Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"e3582937-3e33-4bde-85b2-20a777ce8999","remote_addr":"[fd00:1122:3344:101::b]:42874","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1245,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:51.822911716Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:41055"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:51.934907901Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"e7b2f468-82bc-47c4-b3ff-567f68d35c9b","remote_addr":"[fd00:1122:3344:101::b]:41055","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":111911,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:51.936951743Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"c12b9442-ca76-445f-86b4-e7e1f40ad8d9","remote_addr":"[fd00:1122:3344:101::b]:41055","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1701,"response_code":"200"} {"msg":"Attempting to update network config with generation: 2, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:51.972599718Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:51.973181443Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:51.974581789Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 2","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:51.975554024Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:51.975573044Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"PUT","req_id":"ba6ae336-01f3-4649-a269-a47f63a1b296","remote_addr":"[fd00:1122:3344:101::b]:41055","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":3843,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:52.201003771Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:48208"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:52.201192322Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"75360eaa-25e7-4fc3-8ae7-628b6c7b5619","remote_addr":"[fd00:1122:3344:101::a]:48208","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":180,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:52.252570601Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:38571"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:52.252740486Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"f0d21634-e7ac-4727-805a-0f7dd7e7ec37","remote_addr":"[fd00:1122:3344:101::c]:38571","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":128,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:52.502109906Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:35132"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:52.554777479Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"c07028da-bc27-49c9-94f2-88f9090f64db","remote_addr":"[fd00:1122:3344:101::a]:35132","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":52530,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:52.55520976Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"86ee4745-90a5-4c16-b609-39e3fa5b7ab8","remote_addr":"[fd00:1122:3344:101::a]:35132","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":144,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:52.583104818Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:52193"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:52.630902473Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"d72c8b15-c1ab-4e26-a87e-c39b9fdfff6b","remote_addr":"[fd00:1122:3344:101::c]:52193","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":47723,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:29:52.631231771Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"92f20d9e-9b22-411e-8a54-c02849376ad7","remote_addr":"[fd00:1122:3344:101::c]:52193","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":106,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:30:02.255374648Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:51658"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:30:02.255467741Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"e1fd8d30-44aa-4d9a-bcb7-7f1a073c139f","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:51658","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":68,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:30:21.274223064Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:52395"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:30:21.275668641Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"eaa811f1-a9fc-48cc-9302-1b9fac472542","remote_addr":"[fd00:1122:3344:101::b]:52395","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1415,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:30:21.382835084Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:46675"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:30:21.430031832Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"6876e247-1a1a-4dfa-a617-887527506790","remote_addr":"[fd00:1122:3344:101::b]:46675","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":47123,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:30:21.43032978Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"7fbcf9e3-78b5-48d9-87bd-d6e97d6542db","remote_addr":"[fd00:1122:3344:101::b]:46675","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":100,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:30:22.006368644Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:44017"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:30:22.00660843Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"92ec99ef-0e76-471a-b658-99833704d0ca","remote_addr":"[fd00:1122:3344:101::a]:44017","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":225,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:30:22.027334617Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:58733"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:30:22.027503401Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"865ad3ec-b0be-4b29-91ce-a95200503e0f","remote_addr":"[fd00:1122:3344:101::c]:58733","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":93,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:30:22.166346809Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:57989"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:30:22.21798107Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"231cc304-10f0-4dcc-ba5a-e36a3e27aa45","remote_addr":"[fd00:1122:3344:101::c]:57989","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":51572,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:30:22.218161305Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:45742"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:30:22.221703545Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"2f0028cf-bc2d-43bb-9455-8aaf3619197a","remote_addr":"[fd00:1122:3344:101::c]:57989","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":3395,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:30:22.26717014Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"47bc9d49-45ad-454f-ab26-4e09c7c8b0e5","remote_addr":"[fd00:1122:3344:101::a]:45742","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":48922,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:30:22.267520629Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"d21b89af-278c-44e9-a609-9b4a23b81841","remote_addr":"[fd00:1122:3344:101::a]:45742","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":148,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:30:32.28890768Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:51197"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:30:32.289025033Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"5ab39985-418e-42ba-9688-0ee75ee49399","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:51197","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":93,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:30:51.45824197Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:55380"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:30:51.459701197Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"9d0443a4-b5f7-4a1a-bb21-221389f9be5d","remote_addr":"[fd00:1122:3344:101::b]:55380","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1426,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:30:51.700546805Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:53203"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:30:51.748024502Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"94d01447-0c43-4047-91d3-6ce485071cbd","remote_addr":"[fd00:1122:3344:101::b]:53203","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":47408,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:30:51.748271838Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"a7935cf4-59a9-4ae5-94a1-29cc36eda1ba","remote_addr":"[fd00:1122:3344:101::b]:53203","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":100,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:30:52.093673603Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:57164"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:30:52.094835312Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"03c34d4a-3c9e-4664-b0d6-358a04cec6bd","remote_addr":"[fd00:1122:3344:101::a]:57164","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1104,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:30:52.237944548Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:65291"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:30:52.238113242Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"f9df22cd-2656-4cf4-a410-90e996ef024a","remote_addr":"[fd00:1122:3344:101::c]:65291","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":147,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:30:52.353020631Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:62014"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:30:52.401819051Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"03632ee9-368e-481d-b854-8314dadf9065","remote_addr":"[fd00:1122:3344:101::a]:62014","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":48741,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:30:52.402095558Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:37795"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:30:52.402404266Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"9bcda81f-af02-4d00-9305-9ae107d66552","remote_addr":"[fd00:1122:3344:101::a]:62014","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":285,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:30:52.455731831Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"1c08465a-3339-43c4-a307-1708be275e34","remote_addr":"[fd00:1122:3344:101::c]:37795","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":53481,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:30:52.45607435Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"ddf0d33d-2d3b-480b-9e1e-d16a77ceec3e","remote_addr":"[fd00:1122:3344:101::c]:37795","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":121,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:31:02.320606445Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:56011"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:31:02.320728638Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"93e0c58a-1799-4095-ae1a-daae49d47b62","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:56011","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":96,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:31:21.142769836Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:48212"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:31:21.142965521Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"cd1ecccd-72dd-480f-bbe3-d3aadc835c24","remote_addr":"[fd00:1122:3344:101::b]:48212","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":183,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:31:21.293398883Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:63418"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:31:21.344526132Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"34288959-9398-49bc-b613-459e70cc42ac","remote_addr":"[fd00:1122:3344:101::b]:63418","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":51059,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:31:21.34483412Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"1072c50e-27f4-4104-ac3a-f7e91eb5dcd5","remote_addr":"[fd00:1122:3344:101::b]:63418","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":101,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:31:21.999604284Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:48147"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:31:22.000601499Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"ff641ff8-94b1-4ef7-a05e-4e46f9ff1141","remote_addr":"[fd00:1122:3344:101::c]:48147","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":947,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:31:22.036080881Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:44250"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:31:22.036234025Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"dd60ed8a-9a6b-480a-ae70-26f6dd3b1603","remote_addr":"[fd00:1122:3344:101::a]:44250","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":140,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:31:22.082423298Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:61236"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:31:22.13209127Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"6d0a810b-7e53-4245-b624-5f8bfee2bdba","remote_addr":"[fd00:1122:3344:101::c]:61236","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":49609,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:31:22.132922601Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"60878687-7909-430e-bff0-86501d890734","remote_addr":"[fd00:1122:3344:101::c]:61236","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":683,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:31:22.203260638Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:50888"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:31:22.250512868Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"b74f94bf-9e68-4c30-885e-4bfc4e1c333b","remote_addr":"[fd00:1122:3344:101::a]:50888","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":47152,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:31:22.25137326Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"4d03a1e7-1e0e-49be-9d5b-80a379039f32","remote_addr":"[fd00:1122:3344:101::a]:50888","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":726,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:31:32.352168459Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:49014"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:31:32.353103003Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"aff20318-8eeb-4d9d-8de2-9236e7db27e9","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:49014","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":901,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:31:34.082780845Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:39768","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","component":"producer-server","component":"MetricsManager","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::d]:48425"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:31:34.094749579Z","hostname":"sock","pid":1555,"uri":"/8b6fc4f9-f2be-4116-a97c-1830a2193641","method":"GET","req_id":"e3f8697d-237b-46e3-a2c1-aff06b7565a2","remote_addr":"[fd00:1122:3344:101::d]:48425","local_addr":"[fd00:1122:3344:101::1]:39768","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":11891,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:31:51.295349656Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:44568"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:31:51.295585842Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"e1440809-035a-47eb-b866-14b27ed2773a","remote_addr":"[fd00:1122:3344:101::b]:44568","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":218,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:31:51.467605522Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:59755"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:31:51.517306685Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"f25bb00c-c893-4a34-bbdc-fb866a35fa92","remote_addr":"[fd00:1122:3344:101::b]:59755","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":49636,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:31:51.518135986Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"4a582fd7-1efd-47fb-b243-69d16e98e725","remote_addr":"[fd00:1122:3344:101::b]:59755","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":715,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:31:52.072888959Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:50617"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:31:52.073121385Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"2d657e89-bd07-43bb-97e0-92be9508137e","remote_addr":"[fd00:1122:3344:101::c]:50617","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":210,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:31:52.224372147Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:61642"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:31:52.224507771Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"7a146441-c277-4a3f-a251-d8a2f8b6818b","remote_addr":"[fd00:1122:3344:101::a]:61642","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":113,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:31:52.394625762Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:57867"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:31:52.447222989Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"0713c4bc-2b2d-4a25-a85d-3ed5c004fd19","remote_addr":"[fd00:1122:3344:101::a]:57867","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":52521,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:31:52.447355942Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:55726"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:31:52.450880612Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"182a20f4-1ca3-4aa8-95d2-9a5f1a3cf1d6","remote_addr":"[fd00:1122:3344:101::a]:57867","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":3315,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:31:52.494928931Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"c1393c2e-c925-4e09-8a21-c828bf96b3b3","remote_addr":"[fd00:1122:3344:101::c]:55726","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":47459,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:31:52.49529029Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"9a8081b1-f781-4e1b-8879-8112e8e18713","remote_addr":"[fd00:1122:3344:101::c]:55726","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":127,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:32:02.385046664Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:62685"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:32:02.385147997Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"30bb208c-7fba-45a0-9998-766484538bea","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:62685","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":91,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:32:04.081407846Z","hostname":"sock","pid":1555,"uri":"/8b6fc4f9-f2be-4116-a97c-1830a2193641","method":"GET","req_id":"a3b379ba-7dbe-4524-8ff5-ae829cd3d871","remote_addr":"[fd00:1122:3344:101::d]:48425","local_addr":"[fd00:1122:3344:101::1]:39768","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":769,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:32:21.134173268Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:45936"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:32:21.13502102Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"d466b29f-8cbe-4aa0-a7d2-fedbe0a229ce","remote_addr":"[fd00:1122:3344:101::b]:45936","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":840,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:32:21.255113691Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:60668"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:32:21.303547351Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"7ded5747-a1d9-474b-bdbf-a036c88dc584","remote_addr":"[fd00:1122:3344:101::b]:60668","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":48365,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:32:21.303840009Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"d7c5f85e-d129-421b-8273-9533109247a3","remote_addr":"[fd00:1122:3344:101::b]:60668","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":101,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:32:21.970816494Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:56724"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:32:21.971048879Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"c4bc5104-cf46-4ead-bbef-f74d7cb58a63","remote_addr":"[fd00:1122:3344:101::c]:56724","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":181,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:32:21.994509395Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:59640"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:32:21.994658409Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"63e47803-12ec-485c-829b-339b5846bfcd","remote_addr":"[fd00:1122:3344:101::a]:59640","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":111,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:32:22.10843055Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:63842"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:32:22.158687766Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"93741ba6-27bb-434a-8cee-360b3459e05a","remote_addr":"[fd00:1122:3344:101::c]:63842","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":50182,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:32:22.158801309Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:36307"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:32:22.162838052Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"83b3ae06-8e0e-4f61-a4a0-01e665f84b75","remote_addr":"[fd00:1122:3344:101::c]:63842","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":3831,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:32:22.202627443Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"39e26eb6-1463-41ae-9912-003be55eda74","remote_addr":"[fd00:1122:3344:101::a]:36307","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":43736,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:32:22.203007282Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"f6d018f5-6f5d-4975-8ce2-1e130f5e6e70","remote_addr":"[fd00:1122:3344:101::a]:36307","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":164,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:32:32.413337942Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:58405"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:32:32.413434774Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"569d3f35-392f-4a60-9c75-ba8383933dd8","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:58405","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":74,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:32:34.078993898Z","hostname":"sock","pid":1555,"uri":"/8b6fc4f9-f2be-4116-a97c-1830a2193641","method":"GET","req_id":"9840d5ed-4027-4cb4-b137-b760bdd526dc","remote_addr":"[fd00:1122:3344:101::d]:48425","local_addr":"[fd00:1122:3344:101::1]:39768","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":535,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:32:51.372307991Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:49649"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:32:51.372507966Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"31a76f38-49b1-4179-b47b-ba48d1d6eabc","remote_addr":"[fd00:1122:3344:101::b]:49649","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":174,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:32:51.57222761Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:51586"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:32:51.631330142Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"755ad04a-aa21-4ddb-bc5a-4f9c0da95702","remote_addr":"[fd00:1122:3344:101::b]:51586","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":59021,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:32:51.631753552Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"3130f9c1-fcc0-4c5b-95fd-44940ebb7e0f","remote_addr":"[fd00:1122:3344:101::b]:51586","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":166,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:32:52.183704725Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:50231"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:32:52.18390283Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"64785fa0-60d1-4f14-98a7-b818088568b9","remote_addr":"[fd00:1122:3344:101::c]:50231","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":186,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:32:52.214054866Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:50468"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:32:52.214181059Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"d09cc3bc-5ac9-4b80-97b7-bc8d3d9fa149","remote_addr":"[fd00:1122:3344:101::a]:50468","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":105,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:32:52.365544454Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:49126"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:32:52.414348734Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"1147597f-ff38-49f0-a7e9-9970c916ae58","remote_addr":"[fd00:1122:3344:101::c]:49126","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":48753,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:32:52.414601321Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"6229522f-e6b9-4d66-91f2-b03fbca8fd80","remote_addr":"[fd00:1122:3344:101::c]:49126","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":96,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:32:52.440019077Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:61188"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:32:52.481945242Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"5c0d0dd5-4eda-43e1-851b-4d561001334e","remote_addr":"[fd00:1122:3344:101::a]:61188","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":41830,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:32:52.482194828Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"001942d1-1d47-4d65-a05b-5656dced8848","remote_addr":"[fd00:1122:3344:101::a]:61188","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":101,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:02.444844587Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:53772"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:02.44496604Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"6d95a27b-f1e8-48f1-bd42-004af7579c02","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:53772","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":104,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:04.079134876Z","hostname":"sock","pid":1555,"uri":"/8b6fc4f9-f2be-4116-a97c-1830a2193641","method":"GET","req_id":"ebf1c96b-1513-4fe3-994a-7eeefce3798c","remote_addr":"[fd00:1122:3344:101::d]:48425","local_addr":"[fd00:1122:3344:101::1]:39768","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":814,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:21.226042982Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:36217"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:21.226622156Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"5edb4763-d44d-407f-b7c6-58568edcba89","remote_addr":"[fd00:1122:3344:101::b]:36217","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":567,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:21.309176923Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:34478"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:21.359210205Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"e7106fda-c44c-4921-bc49-0bfd447017bb","remote_addr":"[fd00:1122:3344:101::b]:34478","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":49974,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:21.359494792Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"f5b0182c-5c87-46a8-bf44-ea830560de69","remote_addr":"[fd00:1122:3344:101::b]:34478","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":95,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:21.991757444Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:39813"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:21.99198832Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"f51cc630-cb7e-437c-8a01-1cc8264c0c4a","remote_addr":"[fd00:1122:3344:101::a]:39813","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":199,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:21.996261378Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:54195"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:21.996397742Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"dde5cbd8-73ca-46e6-b309-5af9c3b8bd3c","remote_addr":"[fd00:1122:3344:101::c]:54195","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":89,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:22.117904919Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:44749"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:22.167859778Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"bc29fe6a-4c19-4cf6-9794-61359a5c6d8c","remote_addr":"[fd00:1122:3344:101::c]:44749","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":49900,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:22.168081463Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"b2d3dfdb-14da-4308-af6a-d931bd7f58f6","remote_addr":"[fd00:1122:3344:101::c]:44749","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":93,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:22.192321489Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:43839"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:22.238475952Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"5f7562af-c33e-4ce1-b338-0eb4dcfe678b","remote_addr":"[fd00:1122:3344:101::a]:43839","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":46061,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:22.238766159Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"cc5f9184-2b7a-40cb-bbc6-db833c6a5031","remote_addr":"[fd00:1122:3344:101::a]:43839","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":126,"response_code":"200"} {"msg":"running automatic periodic zone bundle cleanup","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:25.589260858Z","hostname":"sock","pid":1555,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:300"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:25.589881644Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:25.626242757Z","hostname":"sock","pid":1555,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:289","time_to_next_cleanup":"599.99999983s","next_cleanup":"Instant { tv_sec: 1812, tv_nsec: 353430300 }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:32.476089968Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:57386"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:32.476245482Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"8759c9b5-44d8-4ff1-8318-a87d7daa374d","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:57386","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":121,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:34.087456335Z","hostname":"sock","pid":1555,"uri":"/8b6fc4f9-f2be-4116-a97c-1830a2193641","method":"GET","req_id":"686bc2b8-1836-494f-997c-c85ee3556286","remote_addr":"[fd00:1122:3344:101::d]:48425","local_addr":"[fd00:1122:3344:101::1]:39768","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":276,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:39.289956123Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:51.220439596Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:51777"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:51.221265847Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"a6255619-82a9-4e02-82dd-98f85478e1cf","remote_addr":"[fd00:1122:3344:101::b]:51777","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":797,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:51.468600761Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:56734"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:51.469623417Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 08ef7d24-0a5e-4f66-b10f-e9a99f4f350e, kind: Service { id: e3ef572f-48a3-41b6-9bbd-9c7f7f828b14 }, name: Name(\"external-dns-e3ef572f-48a3-41b6-9bbd-9c7f7f828b14\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 66])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 14d2a703-40c9-467c-a24d-276aa20f3e98, kind: Service { id: 50c4b53f-46eb-4906-832f-6fe24fcc1c27 }, name: Name(\"external-dns-50c4b53f-46eb-4906-832f-6fe24fcc1c27\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 158])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 342b451a-1b71-482c-ba92-ff847d794986, kind: Service { id: 65b469d2-6eb7-4e46-8276-ad6d27b67bf9 }, name: Name(\"nexus-65b469d2-6eb7-4e46-8276-ad6d27b67bf9\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 36])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 4162f617-779a-4702-98c1-16953936e276, kind: Service { id: 4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c }, name: Name(\"nexus-4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 8])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f7672b04-24b2-4e2f-9016-12a5c768bb8d, kind: Service { id: 8f64d0bd-6f7c-418c-a532-2ff9143ec947 }, name: Name(\"nexus-8f64d0bd-6f7c-418c-a532-2ff9143ec947\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 152, 21])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:51.469681308Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:51.469727759Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:51.46975281Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:51.46977441Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:51.469783701Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:51.469821962Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:51.469845572Z","hostname":"sock","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"05121047-cd5f-41f9-8a91-af3ee44fbe66","remote_addr":"[fd00:1122:3344:101::b]:56734","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1190,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:51.631657673Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:59891"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:51.680019632Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"51dc150a-d51c-45ae-8ba4-c82a8e7cfd60","remote_addr":"[fd00:1122:3344:101::b]:59891","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":48309,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:51.680275438Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"d517f0f1-3bd7-434e-8f68-96b8e124df52","remote_addr":"[fd00:1122:3344:101::b]:59891","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":109,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:52.014075668Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:62115"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:52.014311554Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"14f201cc-66e4-42b6-ba5c-d5f2cb1cfd10","remote_addr":"[fd00:1122:3344:101::c]:62115","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":191,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:52.188426878Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:57587"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:52.189597757Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 08ef7d24-0a5e-4f66-b10f-e9a99f4f350e, kind: Service { id: e3ef572f-48a3-41b6-9bbd-9c7f7f828b14 }, name: Name(\"external-dns-e3ef572f-48a3-41b6-9bbd-9c7f7f828b14\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 66])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 14d2a703-40c9-467c-a24d-276aa20f3e98, kind: Service { id: 50c4b53f-46eb-4906-832f-6fe24fcc1c27 }, name: Name(\"external-dns-50c4b53f-46eb-4906-832f-6fe24fcc1c27\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 158])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 342b451a-1b71-482c-ba92-ff847d794986, kind: Service { id: 65b469d2-6eb7-4e46-8276-ad6d27b67bf9 }, name: Name(\"nexus-65b469d2-6eb7-4e46-8276-ad6d27b67bf9\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 36])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 4162f617-779a-4702-98c1-16953936e276, kind: Service { id: 4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c }, name: Name(\"nexus-4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 8])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f7672b04-24b2-4e2f-9016-12a5c768bb8d, kind: Service { id: 8f64d0bd-6f7c-418c-a532-2ff9143ec947 }, name: Name(\"nexus-8f64d0bd-6f7c-418c-a532-2ff9143ec947\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 152, 21])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:52.189642689Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:52.18967801Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:52.1896899Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:52.18970318Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:52.18970925Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:52.189727221Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:52.189752161Z","hostname":"sock","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"cb415751-bbb1-4afd-8394-aa742d9d3935","remote_addr":"[fd00:1122:3344:101::c]:57587","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1273,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:52.193855986Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:58668"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:52.193988579Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"19a00700-aa6c-49ba-bf02-97bf0c03452f","remote_addr":"[fd00:1122:3344:101::a]:58668","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":96,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:52.312150901Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:59945"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:52.312262094Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 08ef7d24-0a5e-4f66-b10f-e9a99f4f350e, kind: Service { id: e3ef572f-48a3-41b6-9bbd-9c7f7f828b14 }, name: Name(\"external-dns-e3ef572f-48a3-41b6-9bbd-9c7f7f828b14\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 66])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 14d2a703-40c9-467c-a24d-276aa20f3e98, kind: Service { id: 50c4b53f-46eb-4906-832f-6fe24fcc1c27 }, name: Name(\"external-dns-50c4b53f-46eb-4906-832f-6fe24fcc1c27\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 158])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 342b451a-1b71-482c-ba92-ff847d794986, kind: Service { id: 65b469d2-6eb7-4e46-8276-ad6d27b67bf9 }, name: Name(\"nexus-65b469d2-6eb7-4e46-8276-ad6d27b67bf9\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 36])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 4162f617-779a-4702-98c1-16953936e276, kind: Service { id: 4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c }, name: Name(\"nexus-4c9d7efe-dded-4bfd-b6fb-594f5dd61d1c\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 8])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f7672b04-24b2-4e2f-9016-12a5c768bb8d, kind: Service { id: 8f64d0bd-6f7c-418c-a532-2ff9143ec947 }, name: Name(\"nexus-8f64d0bd-6f7c-418c-a532-2ff9143ec947\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 152, 21])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:52.312288894Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:52.312305985Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:52.312321705Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:52.312331126Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:52.312336536Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:52.313265419Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:52.31327866Z","hostname":"sock","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"e3a0ec49-f083-4ce1-abb4-6f8208cb525a","remote_addr":"[fd00:1122:3344:101::a]:59945","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1105,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:52.378877976Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:33856"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:52.422388391Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"757bc1e6-db3b-4d3b-8a6b-8e4afe2dfda7","remote_addr":"[fd00:1122:3344:101::c]:33856","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":43469,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:52.422528935Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:64595"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:52.426190198Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"77ef78f0-bf27-4b0d-ab91-ab149a100d16","remote_addr":"[fd00:1122:3344:101::c]:33856","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":3415,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:52.467924999Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"47ccca22-7638-4472-9e75-6f78b4714819","remote_addr":"[fd00:1122:3344:101::a]:64595","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":45305,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:33:52.468240436Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"8b9423e0-0cfe-47b4-8e5f-3d04d0b7c75e","remote_addr":"[fd00:1122:3344:101::a]:64595","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":132,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:34:02.506936909Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:63290"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:34:02.507032842Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"5afb258b-1e19-4ddf-8a3c-d14c9d295d38","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:63290","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":74,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:34:04.150215407Z","hostname":"sock","pid":1555,"uri":"/8b6fc4f9-f2be-4116-a97c-1830a2193641","method":"GET","req_id":"3d4cbf6c-8ac3-4557-8d14-18dfa01b2a28","remote_addr":"[fd00:1122:3344:101::d]:48425","local_addr":"[fd00:1122:3344:101::1]:39768","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":835,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:34:21.265287593Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:41964"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:34:21.265518859Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"4e023805-05fc-416b-b32a-0e4b081cb716","remote_addr":"[fd00:1122:3344:101::b]:41964","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":206,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:34:21.355987047Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:51518"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:34:21.401212256Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"2777b8ac-0c02-44fc-89cc-0012516b840c","remote_addr":"[fd00:1122:3344:101::b]:51518","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":45165,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:34:21.401523854Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"5f8ecbab-19cc-4cd9-9dcc-33ceffb79192","remote_addr":"[fd00:1122:3344:101::b]:51518","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":126,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:34:21.955735514Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:52242"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:34:21.956524574Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"14ec04f0-ae14-4cec-a5a3-53a9cb46d25b","remote_addr":"[fd00:1122:3344:101::c]:52242","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":768,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:34:22.028467402Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:64339"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:34:22.028593455Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"05c9932d-2f24-4349-a1c6-0cf366bc66ea","remote_addr":"[fd00:1122:3344:101::a]:64339","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":110,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:34:22.078509893Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:54492"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:34:22.129643192Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"80877a85-da1a-4226-a195-181e700d6cbf","remote_addr":"[fd00:1122:3344:101::c]:54492","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":51070,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:34:22.129881798Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"7253cbc1-680c-4046-97dd-2660007e393d","remote_addr":"[fd00:1122:3344:101::c]:54492","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":90,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:34:22.152198985Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:47496"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:34:22.202311728Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"77720563-f88f-4f95-8c92-a7a4e1d86aca","remote_addr":"[fd00:1122:3344:101::a]:47496","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":50025,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:34:22.202626206Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"e40e91da-15f3-498d-99bd-7e945d13b31d","remote_addr":"[fd00:1122:3344:101::a]:47496","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":103,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:34:32.538683984Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34007"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:34:32.538832348Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"dbc228c3-7cf5-4452-a51d-53d607b44047","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34007","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":111,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:34:34.151530759Z","hostname":"sock","pid":1555,"uri":"/8b6fc4f9-f2be-4116-a97c-1830a2193641","method":"GET","req_id":"e24de341-0182-4ca2-bfeb-3d251f09bc42","remote_addr":"[fd00:1122:3344:101::d]:48425","local_addr":"[fd00:1122:3344:101::1]:39768","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":2538,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:34:51.38194111Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:37980"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:34:51.382190557Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"87f124e6-6597-4084-b4f0-9bfb186d321d","remote_addr":"[fd00:1122:3344:101::b]:37980","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":231,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:34:51.599223291Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:44901"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:34:51.644126741Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"037e3999-f75a-4dc8-ae33-c5f2afafdfdc","remote_addr":"[fd00:1122:3344:101::b]:44901","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":44832,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:34:51.644340897Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"a638b75d-945a-43c6-a188-ec4ccb932c0c","remote_addr":"[fd00:1122:3344:101::b]:44901","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":83,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:34:52.184832777Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:33500"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:34:52.185087594Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"87bb33b4-669e-474a-acb7-8977640a79df","remote_addr":"[fd00:1122:3344:101::c]:33500","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":200,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:34:52.206524488Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:36664"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:34:52.206655212Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"bc5e29f9-889a-4862-bbb1-149483a8a4fb","remote_addr":"[fd00:1122:3344:101::a]:36664","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":96,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:34:52.331238197Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:47146"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:34:52.381446052Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"4b3b18b8-d7ef-46bd-ab80-a7802c190d8c","remote_addr":"[fd00:1122:3344:101::c]:47146","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":50111,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:34:52.381889134Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"31dc2ff9-8f37-48df-8748-34c7b7ee6461","remote_addr":"[fd00:1122:3344:101::c]:47146","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":82,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:34:52.512978424Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:35848"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:34:52.560799489Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"3d37e2b0-4204-4676-8c96-2149cf6a4710","remote_addr":"[fd00:1122:3344:101::a]:35848","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":47720,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:34:52.562912702Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"3247b3b0-ba01-4d14-b5ad-76f022896c7e","remote_addr":"[fd00:1122:3344:101::a]:35848","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":98,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:02.569659263Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:55306"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:02.569783226Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"7912a313-d4de-4ca1-8a6e-33afdc315c81","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:55306","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":83,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:04.035074441Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:57417"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:04.035291436Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"f3b91eea-c26d-44f5-8751-4f0bd65e2328","remote_addr":"[fd00:1122:3344:101::a]:57417","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":191,"response_code":"200"} {"msg":"Mapping virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:04.039166245Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:631","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 242, 120, 53])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(13871353) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:04.039192116Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"PUT","req_id":"19528562-46f9-4e0a-ae32-b25b8afc0207","remote_addr":"[fd00:1122:3344:101::a]:57417","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":3735,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:04.149045686Z","hostname":"sock","pid":1555,"uri":"/8b6fc4f9-f2be-4116-a97c-1830a2193641","method":"GET","req_id":"7c5e3a7c-b4fc-466b-89e8-d40e6971db01","remote_addr":"[fd00:1122:3344:101::d]:48425","local_addr":"[fd00:1122:3344:101::1]:39768","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"8b6fc4f9-f2be-4116-a97c-1830a2193641","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":506,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:04.21491069Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:43209"} {"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:04.231357308Z","hostname":"sock","pid":1555,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:548","metadata":"InstanceMetadata { silo_id: e5bf86a1-53d4-4daf-a85c-9bd551b404ff, project_id: 1d3d8762-14d8-4b4e-8692-4574d6b44df3 }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-06-20T21:35:03.602883Z }","instance_runtime":"InstanceRuntimeState { propolis_id: Some(6b619fe7-bf72-4412-b86d-46e538a55e68 (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-20T21:35:03.643703049Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 785ee6be-35a2-4ef1-bf0f-30831929becf, kind: Instance { id: 77f40f90-e7a8-46c2-bc8f-cbbc5acacede }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 242, 120, 53])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(13871353), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 785ee6be-35a2-4ef1-bf0f-30831929becf, kind: Instance { id: 77f40f90-e7a8-46c2-bc8f-cbbc5acacede }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 242, 120, 53])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(13871353), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 785ee6be-35a2-4ef1-bf0f-30831929becf, kind: Instance { id: 77f40f90-e7a8-46c2-bc8f-cbbc5acacede }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 242, 120, 53])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(13871353), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(13871353))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 785ee6be-35a2-4ef1-bf0f-30831929becf, kind: Instance { id: 77f40f90-e7a8-46c2-bc8f-cbbc5acacede }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 242, 120, 53])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(13871353), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-df1379361d9\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: eea8c572-0ced-4000-9128-6e0086a48c8b, read_only_parent: Some(Volume { block_size: 512, id: 2ac377ff-a5d4-402b-beaf-1e85fb77ef2c, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: b087576a-5d0e-4f04-9a0a-07001c56969b, key: Some(\"xdLnWaWigBUYEWSjo/bYeXYMhLXZd4pOgqlzXMyfK8w=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::13]:19001\", \"[fd00:1122:3344:101::15]:19001\", \"[fd00:1122:3344:101::12]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: eea8c572-0ced-4000-9128-6e0086a48c8b, key: Some(\"DirrrEPOqSw8fCoYowarSpyTrC4CV4LAwiirQcq0oIM=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::18]:19001\", \"[fd00:1122:3344:101::12]:19002\", \"[fd00:1122:3344:101::1a]:19001\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAARYqzUWNRYAABirNRYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAEmKs1FjUWAAAYqzUWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI3N2Y0MGY5MC1lN2E4LTQ2YzItYmM4Zi1jYmJjNWFjYWNlZGUiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJQ2hLdVp1UGROMUV2WGcvQjFsQnlUS0MxYWJ6bkdIc0lqTms5YktvdnJkMCJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }","propolis_id":"6b619fe7-bf72-4412-b86d-46e538a55e68","instance_id":"77f40f90-e7a8-46c2-bc8f-cbbc5acacede"} {"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:04.23144811Z","hostname":"sock","pid":1555,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:582","instance_id":"77f40f90-e7a8-46c2-bc8f-cbbc5acacede (instance)"} {"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:04.231584974Z","hostname":"sock","pid":1555,"component":"InstanceManager","file":"sled-agent/src/instance.rs:942","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 785ee6be-35a2-4ef1-bf0f-30831929becf, kind: Instance { id: 77f40f90-e7a8-46c2-bc8f-cbbc5acacede }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 242, 120, 53])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(13871353), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 785ee6be-35a2-4ef1-bf0f-30831929becf, kind: Instance { id: 77f40f90-e7a8-46c2-bc8f-cbbc5acacede }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 242, 120, 53])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(13871353), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 785ee6be-35a2-4ef1-bf0f-30831929becf, kind: Instance { id: 77f40f90-e7a8-46c2-bc8f-cbbc5acacede }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 242, 120, 53])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(13871353), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(13871353))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 785ee6be-35a2-4ef1-bf0f-30831929becf, kind: Instance { id: 77f40f90-e7a8-46c2-bc8f-cbbc5acacede }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 242, 120, 53])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(13871353), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-df1379361d9\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: eea8c572-0ced-4000-9128-6e0086a48c8b, read_only_parent: Some(Volume { block_size: 512, id: 2ac377ff-a5d4-402b-beaf-1e85fb77ef2c, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: b087576a-5d0e-4f04-9a0a-07001c56969b, key: Some(\"xdLnWaWigBUYEWSjo/bYeXYMhLXZd4pOgqlzXMyfK8w=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::13]:19001\", \"[fd00:1122:3344:101::15]:19001\", \"[fd00:1122:3344:101::12]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: eea8c572-0ced-4000-9128-6e0086a48c8b, key: Some(\"DirrrEPOqSw8fCoYowarSpyTrC4CV4LAwiirQcq0oIM=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::18]:19001\", \"[fd00:1122:3344:101::12]:19002\", \"[fd00:1122:3344:101::1a]:19001\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAARYqzUWNRYAABirNRYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAEmKs1FjUWAAAYqzUWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI3N2Y0MGY5MC1lN2E4LTQ2YzItYmM4Zi1jYmJjNWFjYWNlZGUiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJQ2hLdVp1UGROMUV2WGcvQjFsQnlUS0MxYWJ6bkdIc0lqTms5YktvdnJkMCJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }, instance_runtime: InstanceRuntimeState { propolis_id: Some(6b619fe7-bf72-4412-b86d-46e538a55e68 (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-20T21:35:03.643703049Z }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-06-20T21:35:03.602883Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400 }","propolis_id":"6b619fe7-bf72-4412-b86d-46e538a55e68","instance_id":"77f40f90-e7a8-46c2-bc8f-cbbc5acacede"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:04.232999679Z","hostname":"sock","pid":1555,"uri":"/instances/77f40f90-e7a8-46c2-bc8f-cbbc5acacede","method":"PUT","req_id":"6a7a83d2-dd59-472e-bfe1-b71153051a79","remote_addr":"[fd00:1122:3344:101::a]:43209","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":18011,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:04.39048824Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:34956"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:04.391849905Z","hostname":"sock","pid":1555,"uri":"/instances/77f40f90-e7a8-46c2-bc8f-cbbc5acacede/state","method":"PUT","req_id":"6bc2eaad-14ac-4c53-a65c-207a4de00780","remote_addr":"[fd00:1122:3344:101::a]:34956","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1287,"response_code":"200"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:04.431166244Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 242, 120, 53]), slot: 0, vni: Vni { inner: 13871353 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 }, vnic: \"vopte6\" } }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:04.431437791Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_propolis-server_6b619fe7-bf72-4412-b86d-46e538a55e68","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:04.467555628Z","hostname":"sock","pid":1555,"instance_id":"77f40f90-e7a8-46c2-bc8f-cbbc5acacede","component":"InstanceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_propolis-server_6b619fe7-bf72-4412-b86d-46e538a55e68","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:04.489576208Z","hostname":"sock","pid":1555,"instance_id":"77f40f90-e7a8-46c2-bc8f-cbbc5acacede","component":"InstanceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_propolis-server_6b619fe7-bf72-4412-b86d-46e538a55e68:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:05.9204707Z","hostname":"sock","pid":1555,"instance_id":"77f40f90-e7a8-46c2-bc8f-cbbc5acacede","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:05.920707276Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_6b619fe7-bf72-4412-b86d-46e538a55e68","instance_id":"77f40f90-e7a8-46c2-bc8f-cbbc5acacede","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_propolis-server_6b619fe7-bf72-4412-b86d-46e538a55e68\") failed: Property not found. retry in 35.837679ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:35:06.117901556Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_6b619fe7-bf72-4412-b86d-46e538a55e68","instance_id":"77f40f90-e7a8-46c2-bc8f-cbbc5acacede","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_6b619fe7-bf72-4412-b86d-46e538a55e68\") failed: Property not found. retry in 79.065291ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:35:06.16256373Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_6b619fe7-bf72-4412-b86d-46e538a55e68","instance_id":"77f40f90-e7a8-46c2-bc8f-cbbc5acacede","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_6b619fe7-bf72-4412-b86d-46e538a55e68\") failed: Property not found. retry in 104.039115ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:35:06.256280631Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_6b619fe7-bf72-4412-b86d-46e538a55e68","instance_id":"77f40f90-e7a8-46c2-bc8f-cbbc5acacede","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_6b619fe7-bf72-4412-b86d-46e538a55e68\") failed: Property not found. retry in 473.331777ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:35:06.37312172Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_6b619fe7-bf72-4412-b86d-46e538a55e68","instance_id":"77f40f90-e7a8-46c2-bc8f-cbbc5acacede","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_6b619fe7-bf72-4412-b86d-46e538a55e68\") failed: Property not found. retry in 775.588063ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:35:06.861001464Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_6b619fe7-bf72-4412-b86d-46e538a55e68","instance_id":"77f40f90-e7a8-46c2-bc8f-cbbc5acacede","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_6b619fe7-bf72-4412-b86d-46e538a55e68\") failed: Property not found. retry in 842.749378ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:35:07.646645014Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_6b619fe7-bf72-4412-b86d-46e538a55e68","instance_id":"77f40f90-e7a8-46c2-bc8f-cbbc5acacede","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_6b619fe7-bf72-4412-b86d-46e538a55e68\") failed: Property not found. retry in 1.244862813s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:35:08.500757722Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_6b619fe7-bf72-4412-b86d-46e538a55e68","instance_id":"77f40f90-e7a8-46c2-bc8f-cbbc5acacede","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_6b619fe7-bf72-4412-b86d-46e538a55e68\") failed: Property not found. retry in 961.57138ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:35:09.7559225Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_6b619fe7-bf72-4412-b86d-46e538a55e68","instance_id":"77f40f90-e7a8-46c2-bc8f-cbbc5acacede","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_6b619fe7-bf72-4412-b86d-46e538a55e68\") failed: Property not found. retry in 1.294727679s","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:35:10.727173005Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_6b619fe7-bf72-4412-b86d-46e538a55e68","instance_id":"77f40f90-e7a8-46c2-bc8f-cbbc5acacede","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Started propolis in zone: oxz_propolis-server_6b619fe7-bf72-4412-b86d-46e538a55e68","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:12.048163784Z","hostname":"sock","pid":1555,"instance_id":"77f40f90-e7a8-46c2-bc8f-cbbc5acacede","component":"InstanceManager","file":"sled-agent/src/instance.rs:1427"} {"msg":"wait for service Some(\"oxz_propolis-server_6b619fe7-bf72-4412-b86d-46e538a55e68\") failed: Property not found. retry in 55.314112ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:35:12.057312986Z","hostname":"sock","pid":1555,"instance_id":"77f40f90-e7a8-46c2-bc8f-cbbc5acacede","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_6b619fe7-bf72-4412-b86d-46e538a55e68\") failed: Property not found. retry in 101.374529ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:35:12.121677891Z","hostname":"sock","pid":1555,"instance_id":"77f40f90-e7a8-46c2-bc8f-cbbc5acacede","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:12.233682287Z","hostname":"sock","pid":1555,"instance_id":"77f40f90-e7a8-46c2-bc8f-cbbc5acacede","component":"InstanceManager","file":"sled-agent/src/instance.rs:1436"} {"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:12.27395986Z","hostname":"sock","pid":1555,"instance_id":"77f40f90-e7a8-46c2-bc8f-cbbc5acacede","component":"InstanceManager","file":"sled-agent/src/instance.rs:1450"} {"msg":"Sending ensure request to propolis: InstanceEnsureRequest { cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAARYqzUWNRYAABirNRYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAEmKs1FjUWAAAYqzUWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI3N2Y0MGY5MC1lN2E4LTQ2YzItYmM4Zi1jYmJjNWFjYWNlZGUiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJQ2hLdVp1UGROMUV2WGcvQjFsQnlUS0MxYWJ6bkdIc0lqTms5YktvdnJkMCJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"), disks: [DiskRequest { device: \"nvme\", name: \"disk-df1379361d9\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: eea8c572-0ced-4000-9128-6e0086a48c8b, read_only_parent: Some(Volume { block_size: 512, id: 2ac377ff-a5d4-402b-beaf-1e85fb77ef2c, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: b087576a-5d0e-4f04-9a0a-07001c56969b, key: Some(\"xdLnWaWigBUYEWSjo/bYeXYMhLXZd4pOgqlzXMyfK8w=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::13]:19001\", \"[fd00:1122:3344:101::15]:19001\", \"[fd00:1122:3344:101::12]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: eea8c572-0ced-4000-9128-6e0086a48c8b, key: Some(\"DirrrEPOqSw8fCoYowarSpyTrC4CV4LAwiirQcq0oIM=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::18]:19001\", \"[fd00:1122:3344:101::12]:19002\", \"[fd00:1122:3344:101::1a]:19001\"] } }] } }], migrate: None, nics: [NetworkInterfaceRequest { name: \"vopte6\", slot: Slot(0) }], properties: InstanceProperties { bootrom_id: 00000000-0000-0000-0000-000000000000, description: \"Test description\", id: 77f40f90-e7a8-46c2-bc8f-cbbc5acacede, image_id: 00000000-0000-0000-0000-000000000000, memory: 1024, metadata: InstanceMetadata { project_id: 1d3d8762-14d8-4b4e-8692-4574d6b44df3, silo_id: e5bf86a1-53d4-4daf-a85c-9bd551b404ff }, name: \"localshark\", vcpus: 2 } }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:12.274435342Z","hostname":"sock","pid":1555,"instance_id":"77f40f90-e7a8-46c2-bc8f-cbbc5acacede","component":"InstanceManager","file":"sled-agent/src/instance.rs:689"} {"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:12.389636539Z","hostname":"sock","pid":1555,"instance_id":"77f40f90-e7a8-46c2-bc8f-cbbc5acacede","component":"InstanceManager","file":"sled-agent/src/instance.rs:691"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Starting), migration_status: NoMigration, time: 2024-06-20T21:35:12.390095761Z }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:12.390281075Z","hostname":"sock","pid":1555,"instance_id":"77f40f90-e7a8-46c2-bc8f-cbbc5acacede","component":"InstanceManager","file":"sled-agent/src/instance.rs:573"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:12.390298086Z","hostname":"sock","pid":1555,"instance_id":"77f40f90-e7a8-46c2-bc8f-cbbc5acacede","component":"InstanceManager","file":"sled-agent/src/instance.rs:595","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-06-20T21:35:12.390095761Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(6b619fe7-bf72-4412-b86d-46e538a55e68 (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-20T21:35:03.643703049Z }","propolis_id":"6b619fe7-bf72-4412-b86d-46e538a55e68"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:12.390306856Z","hostname":"sock","pid":1555,"instance_id":"77f40f90-e7a8-46c2-bc8f-cbbc5acacede","component":"InstanceManager","file":"sled-agent/src/instance.rs:492","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(6b619fe7-bf72-4412-b86d-46e538a55e68 (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-20T21:35:03.643703049Z }, propolis_id: 6b619fe7-bf72-4412-b86d-46e538a55e68 (propolis), vmm_state: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-06-20T21:35:12.390095761Z }, migration_state: None }","instance_id":"77f40f90-e7a8-46c2-bc8f-cbbc5acacede"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_status: NoMigration, time: 2024-06-20T21:35:12.514939102Z }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:12.514959943Z","hostname":"sock","pid":1555,"instance_id":"77f40f90-e7a8-46c2-bc8f-cbbc5acacede","component":"InstanceManager","file":"sled-agent/src/instance.rs:573"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:12.514987164Z","hostname":"sock","pid":1555,"instance_id":"77f40f90-e7a8-46c2-bc8f-cbbc5acacede","component":"InstanceManager","file":"sled-agent/src/instance.rs:595","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-06-20T21:35:12.514939102Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(6b619fe7-bf72-4412-b86d-46e538a55e68 (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-20T21:35:03.643703049Z }","propolis_id":"6b619fe7-bf72-4412-b86d-46e538a55e68"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:12.514995544Z","hostname":"sock","pid":1555,"instance_id":"77f40f90-e7a8-46c2-bc8f-cbbc5acacede","component":"InstanceManager","file":"sled-agent/src/instance.rs:492","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(6b619fe7-bf72-4412-b86d-46e538a55e68 (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-20T21:35:03.643703049Z }, propolis_id: 6b619fe7-bf72-4412-b86d-46e538a55e68 (propolis), vmm_state: VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-06-20T21:35:12.514939102Z }, migration_state: None }","instance_id":"77f40f90-e7a8-46c2-bc8f-cbbc5acacede"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:21.201613605Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:48187"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:21.202442457Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"bb4ebc0b-7c92-4deb-8d40-77b0f351490f","remote_addr":"[fd00:1122:3344:101::b]:48187","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":819,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:21.327537925Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:52128"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:21.328192261Z","hostname":"sock","pid":1555,"uri":"/instances/77f40f90-e7a8-46c2-bc8f-cbbc5acacede/state","method":"GET","req_id":"ce62ebb0-36a1-4e10-81a1-9171b7fa80b0","remote_addr":"[fd00:1122:3344:101::b]:52128","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":631,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:21.426634482Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:49149"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:21.474055887Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"44311f55-2f92-4801-8310-f86839ca29ff","remote_addr":"[fd00:1122:3344:101::b]:49149","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":47320,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:21.474375935Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"83cf00b1-3dd7-4e2c-8754-f93966caffa0","remote_addr":"[fd00:1122:3344:101::b]:49149","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":108,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:21.629800994Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:45241"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:21.631343213Z","hostname":"sock","pid":1555,"uri":"/instances/77f40f90-e7a8-46c2-bc8f-cbbc5acacede/state","method":"PUT","req_id":"a2dd01dd-981f-4f6c-80ee-48622a66450c","remote_addr":"[fd00:1122:3344:101::a]:45241","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1458,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_status: NoMigration, time: 2024-06-20T21:35:21.631448896Z }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:21.631476636Z","hostname":"sock","pid":1555,"instance_id":"77f40f90-e7a8-46c2-bc8f-cbbc5acacede","component":"InstanceManager","file":"sled-agent/src/instance.rs:573"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:21.631499257Z","hostname":"sock","pid":1555,"instance_id":"77f40f90-e7a8-46c2-bc8f-cbbc5acacede","component":"InstanceManager","file":"sled-agent/src/instance.rs:595","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-06-20T21:35:21.631448896Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(6b619fe7-bf72-4412-b86d-46e538a55e68 (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-20T21:35:03.643703049Z }","propolis_id":"6b619fe7-bf72-4412-b86d-46e538a55e68"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:21.631519647Z","hostname":"sock","pid":1555,"instance_id":"77f40f90-e7a8-46c2-bc8f-cbbc5acacede","component":"InstanceManager","file":"sled-agent/src/instance.rs:492","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(6b619fe7-bf72-4412-b86d-46e538a55e68 (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-20T21:35:03.643703049Z }, propolis_id: 6b619fe7-bf72-4412-b86d-46e538a55e68 (propolis), vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-06-20T21:35:21.631448896Z }, migration_state: None }","instance_id":"77f40f90-e7a8-46c2-bc8f-cbbc5acacede"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_status: NoMigration, time: 2024-06-20T21:35:21.799517795Z }","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:21.799551026Z","hostname":"sock","pid":1555,"instance_id":"77f40f90-e7a8-46c2-bc8f-cbbc5acacede","component":"InstanceManager","file":"sled-agent/src/instance.rs:573"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:21.799594357Z","hostname":"sock","pid":1555,"instance_id":"77f40f90-e7a8-46c2-bc8f-cbbc5acacede","component":"InstanceManager","file":"sled-agent/src/instance.rs:595","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(6), time_updated: 2024-06-20T21:35:21.799517795Z }","new_instance_state":"InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-06-20T21:35:21.799517795Z }","propolis_id":"6b619fe7-bf72-4412-b86d-46e538a55e68"} {"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:21.799651449Z","hostname":"sock","pid":1555,"instance_id":"77f40f90-e7a8-46c2-bc8f-cbbc5acacede","component":"InstanceManager","file":"sled-agent/src/instance.rs:610","instance_id":"77f40f90-e7a8-46c2-bc8f-cbbc5acacede"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:21.799665679Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:21.799961627Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:21.800043019Z","hostname":"sock","pid":1555,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:437","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/10b67551-9fe8-4a7e-aee1-8eed094ef18b/debug/bundle/zone\", \"/pool/int/d4a00029-4475-4b78-929a-0a3975ebfb87/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/68941de4-9d96-43c2-b5bb-c76900595397/crypt/debug\", \"/pool/ext/c28d26f6-634d-4800-9bba-2148c8387fee/crypt/debug\", \"/pool/ext/3dcbe67c-4d44-4c02-82ee-c644a9177203/crypt/debug\", \"/pool/ext/ea2c3cdd-ebf6-434c-a220-e4b6e1180947/crypt/debug\", \"/pool/ext/05b3cfac-441e-4e11-8952-0d7b0cba33c0/crypt/debug\", \"/pool/ext/c3976b41-1b46-4fb3-86b8-14634aafccad/crypt/debug\", \"/pool/ext/dd582fd3-6899-4123-a5b6-2a268c4ed13b/crypt/debug\", \"/pool/ext/a553f0be-01ff-4b59-9e74-27e6ddf33491/crypt/debug\", \"/pool/ext/ff7b7614-aeaa-4dfa-8c5e-e5115ba48aa0/crypt/debug\"] }","zone_name":"oxz_propolis-server_6b619fe7-bf72-4412-b86d-46e538a55e68"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:21.9905907Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:50478"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:21.996605773Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:36841"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:21.99888616Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"453477f9-74f1-40d2-b43f-fff844a440c5","remote_addr":"[fd00:1122:3344:101::c]:36841","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1231,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:22.003247961Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:48393"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:22.026370779Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:46414"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:22.026631485Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"ab561452-57cf-4b17-b5f1-648a819f7fb5","remote_addr":"[fd00:1122:3344:101::a]:46414","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":194,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:22.191828232Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:51995"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:22.242979072Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"710b1929-5b53-491b-976f-b8c689d1434d","remote_addr":"[fd00:1122:3344:101::c]:51995","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":51010,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:22.24331646Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"55e08308-9dcd-405c-8cce-8a979420b7d3","remote_addr":"[fd00:1122:3344:101::c]:51995","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":110,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:22.243408263Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:49095"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:22.289417081Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"c4565fc0-cae3-414a-b290-7140250b9aa5","remote_addr":"[fd00:1122:3344:101::a]:49095","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":45885,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:22.289845332Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"43fdb257-d388-4f32-907e-3cce45102d98","remote_addr":"[fd00:1122:3344:101::a]:49095","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":184,"response_code":"200"} {"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:22.603273175Z","hostname":"sock","pid":1555,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1230","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_6b619fe7-bf72-4412-b86d-46e538a55e68\", bundle_id: e40468d3-0082-4788-b02b-4b810fb4abd7 }, time_created: 2024-06-20T21:35:21.800298895Z, version: 0, cause: TerminatedInstance }"} {"msg":"Halting and removing zone: oxz_propolis-server_6b619fe7-bf72-4412-b86d-46e538a55e68","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:35:22.603349087Z","hostname":"sock","pid":1555,"instance_id":"77f40f90-e7a8-46c2-bc8f-cbbc5acacede","component":"InstanceManager","file":"sled-agent/src/instance.rs:784"} {"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:24.404348241Z","hostname":"sock","pid":1555,"instance_id":"77f40f90-e7a8-46c2-bc8f-cbbc5acacede","component":"InstanceManager","file":"illumos-utils/src/zone.rs:269"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:24.429252123Z","hostname":"sock","pid":1555,"instance_id":"77f40f90-e7a8-46c2-bc8f-cbbc5acacede","component":"InstanceManager","file":"sled-agent/src/instance.rs:492","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-06-20T21:35:21.799517795Z }, propolis_id: 6b619fe7-bf72-4412-b86d-46e538a55e68 (propolis), vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-06-20T21:35:24.429190452Z }, migration_state: None }","instance_id":"77f40f90-e7a8-46c2-bc8f-cbbc5acacede"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:24.526614917Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:37531"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:24.526898274Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"29c3bb99-4551-4e9b-a1f6-0ceec305a5b4","remote_addr":"[fd00:1122:3344:101::c]:37531","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":211,"response_code":"200"} {"msg":"Mapping virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:24.527214292Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:631","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 242, 120, 53])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(13871353) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:24.527251563Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"PUT","req_id":"85f94f41-9ba5-469e-810c-a0080d5811f0","remote_addr":"[fd00:1122:3344:101::c]:37531","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":111,"response_code":"204"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:24.795060417Z","hostname":"sock","pid":1555,"instance_id":"77f40f90-e7a8-46c2-bc8f-cbbc5acacede","component":"InstanceManager","file":"sled-agent/src/instance.rs:492","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-06-20T21:35:21.799517795Z }, propolis_id: 6b619fe7-bf72-4412-b86d-46e538a55e68 (propolis), vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-06-20T21:35:24.429190452Z }, migration_state: None }","instance_id":"77f40f90-e7a8-46c2-bc8f-cbbc5acacede"} {"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:24.799306115Z","hostname":"sock","pid":1555,"instance_id":"77f40f90-e7a8-46c2-bc8f-cbbc5acacede","component":"InstanceManager","file":"sled-agent/src/instance.rs:727"} {"msg":"Error handling request","v":0,"name":"SledAgent","level":40,"time":"2024-06-20T21:35:24.81921119Z","hostname":"sock","pid":1555,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:504","err":"Instance(RequestDropped(RecvError(())))","request":"GetState"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:24.819339233Z","hostname":"sock","pid":1555,"uri":"/instances/77f40f90-e7a8-46c2-bc8f-cbbc5acacede/state","method":"GET","req_id":"d8d9830e-ac42-42e0-944c-6fe1bf3fee0d","remote_addr":"[fd00:1122:3344:101::c]:50478","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Error managing instances: Instance Manager dropped our request","latency_us":2828680,"response_code":"500"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:24.819375704Z","hostname":"sock","pid":1555,"uri":"/instances/77f40f90-e7a8-46c2-bc8f-cbbc5acacede/state","method":"GET","req_id":"a1984272-8191-4646-8cc6-1d014c131b5d","remote_addr":"[fd00:1122:3344:101::a]:48393","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Not Found","error_message_internal":"No such instance ID: 77f40f90-e7a8-46c2-bc8f-cbbc5acacede","latency_us":2810957,"response_code":"404"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:25.294031923Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:58482"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:25.294256609Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"52c8d3b7-ced3-4ba2-95fc-726d400ed508","remote_addr":"[fd00:1122:3344:101::a]:58482","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":205,"response_code":"200"} {"msg":"Clearing mapping of virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:25.294546426Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:671","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 242, 120, 53])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(13871353) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-20T21:35:25.294581977Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"DELETE","req_id":"62bfac47-f016-4200-8b97-808f048a2cac","remote_addr":"[fd00:1122:3344:101::a]:58482","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":115,"response_code":"204"}