[ Aug 27 19:45:31 Enabled. ] [ Aug 27 19:45:31 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Aug 27 19:45:31 Method "start" exited with status 0. ] [ Aug 27 19:45:31 Rereading configuration. ] [ Aug 27 19:45:31 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:31.508281167Z","hostname":"buskin","pid":1562,"file":"illumos-utils/src/running_zone.rs:113"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:31.50842127Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:31.560661583Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:31.560704434Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:31.674175998Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:31.683116134Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:31.690966573Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:32.293116323Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fdb0:18c0:4d0d:9fb2::, width: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:32.430911025Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:126"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:32.430966786Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:138"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:32.431080048Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:151"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:32.431147649Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:520"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:32.431172169Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:173","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:32.43120933Z","hostname":"buskin","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:716"} {"msg":"Generated i86pc baseboard Pc { identifier: \"buskin\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:32.434180518Z","hostname":"buskin","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:623"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:32.434189088Z","hostname":"buskin","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:765"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:32.434199019Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:187"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:32.434206669Z","hostname":"buskin","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:824"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:32.434233999Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:32.43428092Z","hostname":"buskin","pid":1562,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:32.434352531Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:32.434376102Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:32.434393442Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:32.444862043Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:32.444891923Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:371","id":"1920ef9b-1f2c-4f76-9da0-39371846beaa (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:32.545099921Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:307","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 1920ef9b-1f2c-4f76-9da0-39371846beaa (zpool), kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:32.75001913Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:32.750050941Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:32.750058851Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:32.750072871Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:32.750078691Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:32.750096891Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:32.750102242Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:32.758666751Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:32.758691892Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:371","id":"d862b57c-61f8-46cc-abb0-f7417780b9ed (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:32.846237302Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:307","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: d862b57c-61f8-46cc-abb0-f7417780b9ed (zpool), kind: Internal }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.060736498Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_0.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.060785949Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 1920ef9b-1f2c-4f76-9da0-39371846beaa (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d862b57c-61f8-46cc-abb0-f7417780b9ed (zpool), kind: Internal } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.060830189Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.06083722Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.06084355Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.06085013Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.06085677Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_1.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.06087804Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 1920ef9b-1f2c-4f76-9da0-39371846beaa (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d862b57c-61f8-46cc-abb0-f7417780b9ed (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.060899251Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.060904781Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.060909691Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.060926111Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.060930681Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_2.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.060936701Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.060943371Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.060948142Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 1920ef9b-1f2c-4f76-9da0-39371846beaa (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d862b57c-61f8-46cc-abb0-f7417780b9ed (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.060963382Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_3.vdev"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.060968412Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.060972642Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.060975872Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 1920ef9b-1f2c-4f76-9da0-39371846beaa (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d862b57c-61f8-46cc-abb0-f7417780b9ed (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.060991302Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.060995302Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.061006642Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.061010383Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.061048313Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_4.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.061062183Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.061067203Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.061075244Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 1920ef9b-1f2c-4f76-9da0-39371846beaa (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d862b57c-61f8-46cc-abb0-f7417780b9ed (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.061099024Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_5.vdev"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.061102924Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.061108564Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.061112584Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.061116234Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.061124634Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_6.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.061129074Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 1920ef9b-1f2c-4f76-9da0-39371846beaa (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d862b57c-61f8-46cc-abb0-f7417780b9ed (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.061160745Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.061165385Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.061169505Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.061174025Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.061179535Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 1920ef9b-1f2c-4f76-9da0-39371846beaa (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d862b57c-61f8-46cc-abb0-f7417780b9ed (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.061205686Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.061211806Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_7.vdev"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.061215226Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.061219246Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.061224836Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.061229196Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 1920ef9b-1f2c-4f76-9da0-39371846beaa (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d862b57c-61f8-46cc-abb0-f7417780b9ed (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.061266007Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_8.vdev"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.061270567Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.061274787Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.061278807Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.061282437Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.061286637Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 1920ef9b-1f2c-4f76-9da0-39371846beaa (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d862b57c-61f8-46cc-abb0-f7417780b9ed (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.061319137Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.061323788Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.061327108Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:97"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.061336658Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:99"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.061342838Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 1920ef9b-1f2c-4f76-9da0-39371846beaa (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d862b57c-61f8-46cc-abb0-f7417780b9ed (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.061371098Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.061375349Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.061378889Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.061382489Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:211"} {"msg":"No ledger in /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.061385929Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.061396279Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.061405599Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.061417969Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.061424999Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.061431799Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.06145941Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:216"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.0614681Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:231"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.07308515Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:731"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.07312609Z","hostname":"buskin","pid":1562,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:186","time_to_next_cleanup":"599.99994688s","next_cleanup":"Instant { tv_sec: 1172, tv_nsec: 891009183 }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.073176851Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.073182861Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3861"} {"msg":"No ledger in /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.073227012Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.073239942Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.076572247Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:205"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.076676999Z","hostname":"buskin","pid":1562,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:44","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.1.164,\n last: 10.151.1.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.1.164,\n 10.151.1.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.1.100,\n vlan_id: None,\n local_pref: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n lldp: None,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.076886132Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1053"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.076893822Z","hostname":"buskin","pid":1562,"component":"DnsResolver","component":"RSS","file":"internal-dns/src/resolver.rs:73","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.076941733Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"No ledger in /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.076987404Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.076995594Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.077002144Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1113"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.077029044Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"No ledger in /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.077038384Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.077046854Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.077050755Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1154"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.077054265Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:155"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:9fb2::1","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.077057755Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:128"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.077084725Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Writing ledger to /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.077116236Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.077578483Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Sled plan written to storage: Plan {\n rack_id: 1368d3f4-c0fd-4c61-a5b2-a8a3e63e09c5,\n sleds: {\n [fdb0:18c0:4d0d:9fb2::1]:12346: StartSledAgentRequest {\n generation: 0,\n schema_version: 1,\n body: StartSledAgentRequestBody {\n id: 4b1d95e0-2253-4dc0-9406-d601ae11a340,\n rack_id: 1368d3f4-c0fd-4c61-a5b2-a8a3e63e09c5,\n use_trust_quorum: false,\n is_lrtq_learner: false,\n subnet: Ipv6Subnet {\n net: Ipv6Net {\n addr: fd00:1122:3344:101::,\n width: 64,\n },\n },\n },\n },\n },\n config: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.1.164,\n last: 10.151.1.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.1.164,\n 10.151.1.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.1.100,\n vlan_id: None,\n local_pref: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n lldp: None,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.07796806Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:175"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.07798755Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1187"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.0780169Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.07802773Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.078184033Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.078312855Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.078347806Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:217","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 4b1d95e0-2253-4dc0-9406-d601ae11a340, rack_id: 1368d3f4-c0fd-4c61-a5b2-a8a3e63e09c5, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.078467958Z","hostname":"buskin","pid":1562,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:53238","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:66"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 4b1d95e0-2253-4dc0-9406-d601ae11a340, rack_id: 1368d3f4-c0fd-4c61-a5b2-a8a3e63e09c5, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.078577499Z","hostname":"buskin","pid":1562,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:359"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.07858528Z","hostname":"buskin","pid":1562,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:374"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.07862938Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/server.rs:44"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.078651591Z","hostname":"buskin","pid":1562,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:73","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.078662081Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.078665971Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:574"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.078669151Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:101::, width: 64 }"} {"msg":"No ledger in /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.078685731Z","hostname":"buskin","pid":1562,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.078710062Z","hostname":"buskin","pid":1562,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.078714542Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:563"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.078717902Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:591"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.078725062Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:395"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.078772363Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.078778623Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:481","result":"DisksManagementResult { status: [] }"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.080863957Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"Starting switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.080922318Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4540"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.111073131Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.132139545Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.148761286Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.160837404Z","hostname":"buskin","pid":1562,"sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:385"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.160880615Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.160912735Z","hostname":"buskin","pid":1562,"sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:397"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.160983706Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 89.221709ms)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.164352581Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8a3b5d7c-cf3f-425a-aacf-a1c5e4a162a9\", \"content-length\": \"124\", \"date\": \"Tue, 27 Aug 2024 19:45:32 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8a3b5d7c-cf3f-425a-aacf-a1c5e4a162a9\" }"} {"msg":"attempting to create encrypted zvol: name=\"oxi_1920ef9b-1f2c-4f76-9da0-39371846beaa/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.175354491Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"successfully created encrypted zvol: name=\"oxi_1920ef9b-1f2c-4f76-9da0-39371846beaa/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.199439435Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_1920ef9b-1f2c-4f76-9da0-39371846beaa/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.199481606Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.204602579Z","hostname":"buskin","pid":1562,"sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:412"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.204611469Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:126"} {"msg":"Ensuring dataset oxi_1920ef9b-1f2c-4f76-9da0-39371846beaa/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.204615569Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:136"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.253990756Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 73.426411ms)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.254576526Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8d7f05ad-da31-43ff-a86b-b1a97130388d\", \"content-length\": \"124\", \"date\": \"Tue, 27 Aug 2024 19:45:32 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8d7f05ad-da31-43ff-a86b-b1a97130388d\" }"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.278258013Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:170"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.329382238Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 139.501499ms)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.329945678Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3640f853-8766-4bfc-ac45-821b916ba3d3\", \"content-length\": \"124\", \"date\": \"Tue, 27 Aug 2024 19:45:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3640f853-8766-4bfc-ac45-821b916ba3d3\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.470456934Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 214.629963ms)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.470966532Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"04a1b72e-cf72-4cc4-b4c9-80751d94eb86\", \"content-length\": \"124\", \"date\": \"Tue, 27 Aug 2024 19:45:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"04a1b72e-cf72-4cc4-b4c9-80751d94eb86\" }"} {"msg":"Mounting oxi_1920ef9b-1f2c-4f76-9da0-39371846beaa/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.49833549Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:178"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.511602776Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.511675108Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.511708778Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.511723918Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:192"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.686886581Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 146.185331ms)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.687612593Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bdab7fbc-6ebd-4e0f-b108-4562439e04c6\", \"content-length\": \"124\", \"date\": \"Tue, 27 Aug 2024 19:45:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"bdab7fbc-6ebd-4e0f-b108-4562439e04c6\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.835237966Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 345.907204ms)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:33.835971508Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"50010532-d833-4f4e-88c6-cf01853c8f08\", \"content-length\": \"124\", \"date\": \"Tue, 27 Aug 2024 19:45:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"50010532-d833-4f4e-88c6-cf01853c8f08\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:34.183627398Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 177.201447ms)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:34.184049085Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"36fd719f-8a3d-4f2c-9d90-df188d312773\", \"content-length\": \"124\", \"date\": \"Tue, 27 Aug 2024 19:45:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"36fd719f-8a3d-4f2c-9d90-df188d312773\" }"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:34.238376823Z","hostname":"buskin","pid":1562,"sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:90"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:34.244089537Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:46199","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:205","component":"producer-server","sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:205"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:34.244122697Z","hostname":"buskin","pid":1562,"component":"internal-dns-resolver","file":"internal-dns/src/resolver.rs:73","component":"producer-server","sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"internal-dns/src/resolver.rs:73","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:34.244187588Z","hostname":"buskin","pid":1562,"file":"oximeter/producer/src/lib.rs:283","component":"producer-server","sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"oximeter/producer/src/lib.rs:283","interval":"30s","address":"[fd00:1122:3344:101::1]:46199","producer_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:34.24550006Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"229.12866ms"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:34.246335103Z","hostname":"buskin","pid":1562,"component":"VmmReservoirManager","sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:34.246349644Z","hostname":"buskin","pid":1562,"component":"VmmReservoirManager","sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:34.246413345Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:915","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:34.246422405Z","hostname":"buskin","pid":1562,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/src/resolver.rs:73","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:34.362478312Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:34.476107029Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"670.915342ms"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:35.149158687Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"581.36695ms"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:35.732194846Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"1.179528552s"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:36.58485022Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2634"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:36.584902781Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2852"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:36.584908471Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2504"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:36.584913021Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2551"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:36.584917361Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2933"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:36.584921791Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2975"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:36.586901223Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2777"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:36.588268306Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2545"} {"msg":"Profile for oxz_switch:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:36.588372157Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:36.58853499Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 39.582632ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:36.836916739Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 113.725705ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:37.0217653Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:37.022667565Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"2.327351412s"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 290.762943ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:37.194709366Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 478.966059ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:37.531326938Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 424.185245ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:38.111632941Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 969.956447ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:38.720983659Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:39.352482379Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"6.555933393s"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.456091096s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:39.881771369Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.350097814s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:41.490155044Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:43.237939877Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:205"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:43.238009068Z","hostname":"buskin","pid":1562,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:429"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:43.238015958Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Writing ledger to /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:43.238033708Z","hostname":"buskin","pid":1562,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:43.238271572Z","hostname":"buskin","pid":1562,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:43.238482716Z","hostname":"buskin","pid":1562,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:43.238500826Z","hostname":"buskin","pid":1562,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:53238","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:73"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:43.238516547Z","hostname":"buskin","pid":1562,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV2 { rack_subnet: Ipv6Net { addr: fd00:1122:3344:100::, width: 56 }, infra_ip_first: 10.151.1.174, infra_ip_last: 10.151.1.174, ports: [PortConfigV2 { routes: [RouteConfig { destination: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), nexthop: 10.151.1.100, vlan_id: None, local_pref: None }], addresses: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false, lldp: None }], bgp: [], bfd: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:43.238526557Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3861"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:43.238595938Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:108","peer":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:43.238604808Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:43.452087987Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"No ledger in /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/config/rss-service-plan-v4.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:43.452179798Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/config/rss-service-plan-v4.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:43.452186039Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:43.452190028Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:43.452193499Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:43.452199779Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:43.472082073Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4182","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:43.472116064Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4198"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:43.472121504Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:534"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:43.712691106Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:62635"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:43.713199934Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:43.713324656Z","hostname":"buskin","pid":1562,"uri":"/inventory","method":"GET","req_id":"1c28cda2-0283-4847-80da-d7f773c69dcc","remote_addr":"[fd00:1122:3344:101::1]:62635","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":294,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:43.744115859Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:54200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:43.744222831Z","hostname":"buskin","pid":1562,"uri":"/sled-role","method":"GET","req_id":"87f960af-a318-4c72-8481-0b67735f13f7","remote_addr":"[fd00:1122:3344:101::1]:54200","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":65,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:43.744915043Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Writing ledger to /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/config/.rss-service-plan-v4.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:43.744941403Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/config/.rss-service-plan-v4.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:43.745251488Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:43.745555453Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:998"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 68d54ab4-b691-413f-a7a2-97f1ef4686e5, pool_id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: e20ea101-2e19-435a-af2b-d371d972c295, pool_id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: 79eac6bd-23cb-4d2a-8d04-30cf4be0e134, pool_id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 50551948-9f20-4a9f-aa91-1c0e21f96c39, pool_id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 7b7fd7c9-f54f-4921-81b7-3c444a968987, pool_id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: a30a30c7-fba0-48d0-881d-3e7d1560a3fe, pool_id: f00f974c-b392-4668-a102-c63201ffd653 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 7d39b557-ff3e-43dd-9b8a-9b2a567e63d0, pool_id: 462cfa74-565a-4b07-9e7f-532c19a0cdef (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 53411289-c77d-451f-b381-d9e971575f47, pool_id: 060ae63c-2ab8-4f4a-9465-ac7729ee0f85 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: a0ab888a-b1bf-4967-bf5f-20036718ce31, pool_id: 3e23c6bd-257b-4c1d-bcac-865f7970574e (zpool) }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:43.774997144Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:355"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:43.775097706Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:33471"} {"msg":"physical disks ensure","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:43.775426571Z","hostname":"buskin","pid":1562,"sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:830"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 68d54ab4-b691-413f-a7a2-97f1ef4686e5, pool_id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: e20ea101-2e19-435a-af2b-d371d972c295, pool_id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: 79eac6bd-23cb-4d2a-8d04-30cf4be0e134, pool_id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 50551948-9f20-4a9f-aa91-1c0e21f96c39, pool_id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 7b7fd7c9-f54f-4921-81b7-3c444a968987, pool_id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: a30a30c7-fba0-48d0-881d-3e7d1560a3fe, pool_id: f00f974c-b392-4668-a102-c63201ffd653 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 7d39b557-ff3e-43dd-9b8a-9b2a567e63d0, pool_id: 462cfa74-565a-4b07-9e7f-532c19a0cdef (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 53411289-c77d-451f-b381-d9e971575f47, pool_id: 060ae63c-2ab8-4f4a-9465-ac7729ee0f85 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: a0ab888a-b1bf-4967-bf5f-20036718ce31, pool_id: 3e23c6bd-257b-4c1d-bcac-865f7970574e (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:43.775486532Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"No ledger in /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:43.775501572Z","hostname":"buskin","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:43.775510923Z","hostname":"buskin","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:43.775515012Z","hostname":"buskin","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:645"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:43.775529793Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:43.775534043Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:43.775538743Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:43.775542583Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:43.93936706Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:43.939393561Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"d598a515-1659-4717-862f-e7cfd6119c10 (zpool)"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.137532818Z","hostname":"buskin","pid":1562,"component":"VmmReservoirManager","sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.231208559Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:307","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.239166859Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:351","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.239315602Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:353","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.239322182Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:357","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.239328152Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:359","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.239335142Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.239469404Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.239476394Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:371"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/4b1d95e0-2253-4dc0-9406-d601ae11a340): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-08-27T19:45:44.243184985Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.273651873Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_d598a515-1659-4717-862f-e7cfd6119c10/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.28204952Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:414"} {"msg":"Looking for unencrypted datasets in oxp_d598a515-1659-4717-862f-e7cfd6119c10","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.405247723Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:499"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.413192173Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.413237184Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.413243584Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.413250824Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.413254844Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.423233568Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.423262898Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.519624393Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:307","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.527675074Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:351","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.527705525Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:353","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.527712925Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:357","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.527732195Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:359","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.527742776Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.527875078Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.527881848Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:371"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.564283423Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_5b912565-4620-4e7b-8d6f-58182fe5c213/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.572362575Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:414"} {"msg":"Looking for unencrypted datasets in oxp_5b912565-4620-4e7b-8d6f-58182fe5c213","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.694037433Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:499"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.702797446Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.702825597Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.702832737Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.702839437Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.702846587Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.712110829Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.712133869Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.805429534Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:307","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","zpool":"ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.813351373Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:351","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.813391144Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:353","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.813395784Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:357","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.813426824Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:359","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.813432965Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.813546666Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.813550906Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:371"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.847336008Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_7bb5d018-eb30-4a1b-bc97-eb30fe64451c/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.855121986Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:414"} {"msg":"Looking for unencrypted datasets in oxp_7bb5d018-eb30-4a1b-bc97-eb30fe64451c","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.977968053Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:499"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.98573602Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.985765991Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.985771451Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.985777651Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.985792831Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.995853216Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:44.995879766Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.091793983Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:307","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","zpool":"ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.099378567Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:351","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.099412798Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:353","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.099422178Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:357","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.099439168Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:359","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.099448159Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.099593041Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.099599801Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:371"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.135136712Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_9573bdf9-c5ac-4b32-9886-9d0a0abf58b3/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.143186383Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:414"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/4b1d95e0-2253-4dc0-9406-d601ae11a340): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-08-27T19:45:45.246167817Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Looking for unencrypted datasets in oxp_9573bdf9-c5ac-4b32-9886-9d0a0abf58b3","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.2659614Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:499"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.273598835Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.273634075Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.273640906Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.273645555Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.273683756Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.283359104Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.283389455Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.375968748Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:307","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","zpool":"ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.383373359Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:351","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.383413509Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:353","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.38342588Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:357","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.38343357Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:359","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.38344182Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.383548662Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.383555362Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:371"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.419515059Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_6388c771-f50c-4e79-aa18-35304bc6bf0b/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.428156811Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:414"} {"msg":"Looking for unencrypted datasets in oxp_6388c771-f50c-4e79-aa18-35304bc6bf0b","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.549787728Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:499"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.557574306Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.557607416Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.557612767Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.557617747Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.557623187Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.567056831Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.567090601Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"f00f974c-b392-4668-a102-c63201ffd653 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.662091394Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:307","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","zpool":"ZpoolName { id: f00f974c-b392-4668-a102-c63201ffd653 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.668977387Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:351","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.669017587Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:353","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.669026877Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:357","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.669068778Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:359","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.669077138Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.669248171Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.669258711Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:371"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.704804182Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_f00f974c-b392-4668-a102-c63201ffd653/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.712535218Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:414"} {"msg":"Looking for unencrypted datasets in oxp_f00f974c-b392-4668-a102-c63201ffd653","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.837678684Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:499"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.845800226Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.845829947Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.845837567Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.845842657Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.845848537Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.85521029Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.85523552Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"462cfa74-565a-4b07-9e7f-532c19a0cdef (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.949707524Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:307","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","zpool":"ZpoolName { id: 462cfa74-565a-4b07-9e7f-532c19a0cdef (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.957669994Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:351","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.957698995Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:353","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.957706495Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:357","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.957746056Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:359","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.957754776Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.957878468Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.957888418Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:371"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:45.961785372Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"17.978470839s"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:45.994002477Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_462cfa74-565a-4b07-9e7f-532c19a0cdef/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.003492422Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:414"} {"msg":"Looking for unencrypted datasets in oxp_462cfa74-565a-4b07-9e7f-532c19a0cdef","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.131692558Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:499"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.139928632Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.139957133Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.139962733Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.139969293Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.139973313Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.149825804Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.149852454Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"060ae63c-2ab8-4f4a-9465-ac7729ee0f85 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.246481244Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:307","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","zpool":"ZpoolName { id: 060ae63c-2ab8-4f4a-9465-ac7729ee0f85 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.255208066Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:351","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.255238677Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:353","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.255266347Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:357","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.255301728Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:359","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.255312868Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.25542967Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.25543936Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:371"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/4b1d95e0-2253-4dc0-9406-d601ae11a340): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-08-27T19:45:46.259592048Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.290459092Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_060ae63c-2ab8-4f4a-9465-ac7729ee0f85/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.299093364Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:414"} {"msg":"Looking for unencrypted datasets in oxp_060ae63c-2ab8-4f4a-9465-ac7729ee0f85","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.429470614Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:499"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.438035204Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.438069345Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.438077425Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.438082175Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.438085985Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.447945806Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.447975647Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"3e23c6bd-257b-4c1d-bcac-865f7970574e (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.541711929Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:307","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","zpool":"ZpoolName { id: 3e23c6bd-257b-4c1d-bcac-865f7970574e (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.549713009Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:351","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.549743Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:353","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.5497483Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:357","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.54977628Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:359","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.549785061Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.549979574Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.549988934Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:371"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.586001242Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_3e23c6bd-257b-4c1d-bcac-865f7970574e/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.594154026Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:414"} {"msg":"Looking for unencrypted datasets in oxp_3e23c6bd-257b-4c1d-bcac-865f7970574e","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.717641574Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:499"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.726036381Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.726067242Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Writing ledger to /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.726118852Z","hostname":"buskin","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.726230264Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 1920ef9b-1f2c-4f76-9da0-39371846beaa (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d862b57c-61f8-46cc-abb0-f7417780b9ed (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: f00f974c-b392-4668-a102-c63201ffd653 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 462cfa74-565a-4b07-9e7f-532c19a0cdef (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 060ae63c-2ab8-4f4a-9465-ac7729ee0f85 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 3e23c6bd-257b-4c1d-bcac-865f7970574e (zpool), kind: External } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.726282705Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.726288295Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Writing ledger to /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.726439717Z","hostname":"buskin","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"physical disks ensure: Updated storage","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.726683561Z","hostname":"buskin","pid":1562,"sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:834"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.726694032Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"physical disks ensure: Propagating new generation of disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.726708872Z","hostname":"buskin","pid":1562,"sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:860","generation":"Generation(1)"} {"msg":"physical disks ensure: Updated storage monitor","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.726717432Z","hostname":"buskin","pid":1562,"sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:865"} {"msg":"physical disks ensure: Updated zone bundler","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.726722252Z","hostname":"buskin","pid":1562,"sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:870"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.726757393Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:121","gen":"Generation(1)"} {"msg":"physical disks ensure: Updated probes","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.726768753Z","hostname":"buskin","pid":1562,"sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:876"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.726823714Z","hostname":"buskin","pid":1562,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:803","gen":"Generation(1)"} {"msg":"physical disks ensure: Updated instances","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.726835694Z","hostname":"buskin","pid":1562,"sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:881"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.726934576Z","hostname":"buskin","pid":1562,"uri":"/omicron-physical-disks","method":"PUT","req_id":"df3d4f16-5b18-427a-8a4d-2b8c8c2b7a30","remote_addr":"[fd00:1122:3344:101::1]:33471","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":2951664,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21e40459-2dec-4da5-8537-0450d95100c7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.758368799Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.758532992Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:50004"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.758795456Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.758801286Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.809625587Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.809655268Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.860256384Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.860287475Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.90891985Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"No ledger in /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.908961231Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.908968031Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.909012191Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.951281712Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.951316693Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:46.951320933Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_ced4c03d-9859-49c7-a5b2-d6706c87f34e","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:47.007545071Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_21e40459-2dec-4da5-8537-0450d95100c7","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:47.011632737Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_c9fa5d8b-74a4-4a22-951f-5a84f02a69e3","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:47.015973268Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_internal_dns_ced4c03d-9859-49c7-a5b2-d6706c87f34e","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:47.027235003Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_21e40459-2dec-4da5-8537-0450d95100c7","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:47.0319534Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_c9fa5d8b-74a4-4a22-951f-5a84f02a69e3","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:47.036613776Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/4b1d95e0-2253-4dc0-9406-d601ae11a340): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-08-27T19:45:47.262450647Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Profile for oxz_internal_dns_ced4c03d-9859-49c7-a5b2-d6706c87f34e:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:48.52135451Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:48.521455102Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:2::, width: 64 }"} {"msg":"Booting oxz_internal_dns_ced4c03d-9859-49c7-a5b2-d6706c87f34e zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:48.521690635Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_ced4c03d-9859-49c7-a5b2-d6706c87f34e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_internal_dns_21e40459-2dec-4da5-8537-0450d95100c7:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:49.004943692Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_internal_dns_c9fa5d8b-74a4-4a22-951f-5a84f02a69e3:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:49.481278057Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:49.481337168Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:3::, width: 64 }"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:49.481348868Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:1::, width: 64 }"} {"msg":"wait for service Some(\"oxz_internal_dns_ced4c03d-9859-49c7-a5b2-d6706c87f34e\") failed: Property not found. retry in 29.971893ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:49.489476421Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_ced4c03d-9859-49c7-a5b2-d6706c87f34e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_internal_dns_21e40459-2dec-4da5-8537-0450d95100c7 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:49.489515311Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_21e40459-2dec-4da5-8537-0450d95100c7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_internal_dns_c9fa5d8b-74a4-4a22-951f-5a84f02a69e3 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:49.493569428Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_c9fa5d8b-74a4-4a22-951f-5a84f02a69e3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_internal_dns_ced4c03d-9859-49c7-a5b2-d6706c87f34e\") failed: Property not found. retry in 131.312763ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:49.528952946Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_ced4c03d-9859-49c7-a5b2-d6706c87f34e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_21e40459-2dec-4da5-8537-0450d95100c7\") failed: Property not found. retry in 52.954188ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:49.617559204Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_21e40459-2dec-4da5-8537-0450d95100c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_c9fa5d8b-74a4-4a22-951f-5a84f02a69e3\") failed: Property not found. retry in 62.993347ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:49.623709555Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_c9fa5d8b-74a4-4a22-951f-5a84f02a69e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_ced4c03d-9859-49c7-a5b2-d6706c87f34e\") failed: Property not found. retry in 142.725618ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:49.669094966Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_ced4c03d-9859-49c7-a5b2-d6706c87f34e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_21e40459-2dec-4da5-8537-0450d95100c7\") failed: Property not found. retry in 71.028974ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:49.678357968Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_21e40459-2dec-4da5-8537-0450d95100c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_c9fa5d8b-74a4-4a22-951f-5a84f02a69e3\") failed: Property not found. retry in 82.012658ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:49.891233327Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_c9fa5d8b-74a4-4a22-951f-5a84f02a69e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_ced4c03d-9859-49c7-a5b2-d6706c87f34e\") failed: Property not found. retry in 316.965837ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:49.932104185Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_ced4c03d-9859-49c7-a5b2-d6706c87f34e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_21e40459-2dec-4da5-8537-0450d95100c7\") failed: Property not found. retry in 151.868053ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:49.943039883Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_21e40459-2dec-4da5-8537-0450d95100c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_c9fa5d8b-74a4-4a22-951f-5a84f02a69e3\") failed: Property not found. retry in 255.22157ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:50.000119295Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_c9fa5d8b-74a4-4a22-951f-5a84f02a69e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_21e40459-2dec-4da5-8537-0450d95100c7\") failed: Property not found. retry in 296.754859ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:50.11909404Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_21e40459-2dec-4da5-8537-0450d95100c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_ced4c03d-9859-49c7-a5b2-d6706c87f34e\") failed: Property not found. retry in 1.132802269s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:50.25859713Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_ced4c03d-9859-49c7-a5b2-d6706c87f34e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_c9fa5d8b-74a4-4a22-951f-5a84f02a69e3\") failed: Property not found. retry in 473.238566ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:50.266648711Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_c9fa5d8b-74a4-4a22-951f-5a84f02a69e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_21e40459-2dec-4da5-8537-0450d95100c7\") failed: Property not found. retry in 1.103628956s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:50.424898628Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_21e40459-2dec-4da5-8537-0450d95100c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_c9fa5d8b-74a4-4a22-951f-5a84f02a69e3\") failed: Property not found. retry in 935.420318ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:50.750528879Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_c9fa5d8b-74a4-4a22-951f-5a84f02a69e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_ced4c03d-9859-49c7-a5b2-d6706c87f34e\") failed: Property not found. retry in 1.200837455s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:51.401536578Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_ced4c03d-9859-49c7-a5b2-d6706c87f34e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_21e40459-2dec-4da5-8537-0450d95100c7\") failed: Property not found. retry in 962.503044ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:51.539864249Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_21e40459-2dec-4da5-8537-0450d95100c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_c9fa5d8b-74a4-4a22-951f-5a84f02a69e3\") failed: Property not found. retry in 1.406374949s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:51.695741856Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_c9fa5d8b-74a4-4a22-951f-5a84f02a69e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_21e40459-2dec-4da5-8537-0450d95100c7\") failed: Property not found. retry in 1.204685324s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:52.510624743Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_21e40459-2dec-4da5-8537-0450d95100c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_ced4c03d-9859-49c7-a5b2-d6706c87f34e\") failed: Property not found. retry in 799.903207ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:52.610885052Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_ced4c03d-9859-49c7-a5b2-d6706c87f34e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:52.859431544Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4206"} {"msg":"Ensuring there is a default route","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:52.859483575Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4216","gateway":"fd00:1122:3344:101::1"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:52.865383421Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4295"} {"msg":"refreshed dendrite service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:52.990825501Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4343"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:52.990868471Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4370"} {"msg":"refreshed lldpd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:53.033352756Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4383"} {"msg":"configuring MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:53.033390636Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4247"} {"msg":"refreshed MGS service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:53.083758369Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4289"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:53.08379985Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4350","rack_subnet":"fd00:1122:3344::"} {"msg":"refreshed wicketd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:53.105684367Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4361"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:53.105760449Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4403"} {"msg":"wait for service Some(\"oxz_internal_dns_c9fa5d8b-74a4-4a22-951f-5a84f02a69e3\") failed: Property not found. retry in 1.03772493s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:53.112347087Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_c9fa5d8b-74a4-4a22-951f-5a84f02a69e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"refreshed mgd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:53.199554832Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4435"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:53.199599232Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4441"} {"msg":"refreshed mg-ddm service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:53.312374936Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4482"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:53.312433217Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:366"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/4b1d95e0-2253-4dc0-9406-d601ae11a340): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:53.347478789Z","hostname":"buskin","pid":1562,"component":"NexusNotifierTask","sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"sled-agent/src/nexus.rs:571"} {"msg":"Failed to get switch ID from MGS (retrying in 65.243776ms)","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:53.347587101Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:385","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"wait for service Some(\"oxz_internal_dns_ced4c03d-9859-49c7-a5b2-d6706c87f34e\") failed: Property not found. retry in 1.382129123s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:53.419980114Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_ced4c03d-9859-49c7-a5b2-d6706c87f34e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Failed to get switch ID from MGS (retrying in 137.229537ms)","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:53.42033059Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:385","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"Failed to get switch ID from MGS (retrying in 225.509508ms)","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:53.558057681Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:385","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"wait for service Some(\"oxz_internal_dns_21e40459-2dec-4da5-8537-0450d95100c7\") failed: Property not found. retry in 1.140804862s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:53.724519471Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_21e40459-2dec-4da5-8537-0450d95100c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Failed to get switch ID from MGS (retrying in 511.821253ms)","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:53.784848807Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:385","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:53.919577909Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::2]:58017"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:53.920480734Z","hostname":"buskin","pid":1562,"uri":"/sled-identifiers","method":"GET","req_id":"fc7ac97c-90f5-4dcd-b187-d88344579a42","remote_addr":"[fd00:1122:3344:101::2]:58017","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":649,"response_code":"200"} {"msg":"wait for service Some(\"oxz_internal_dns_c9fa5d8b-74a4-4a22-951f-5a84f02a69e3\") failed: Property not found. retry in 882.768753ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:54.159133023Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_c9fa5d8b-74a4-4a22-951f-5a84f02a69e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:54.297920581Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:414"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:54.33331094Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:676"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:54.333426791Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:714"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:54.333912859Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:717","uptime":"1"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:54.33392766Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:436","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.1.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n}"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:54.375447338Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:54.427870215Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:54.427930056Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:54.427938286Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:54.427946046Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:54.427951946Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:54.427958956Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:54.427964747Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:54.427972097Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:54.427977927Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:54.427993707Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:54.428000027Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:54.428008557Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:54.428018128Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:54.428025977Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:54.428035138Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:54.428041338Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:54.428047848Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:54.428054108Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:54.428060638Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:54.428066778Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:54.428073038Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:54.428090369Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:54.428096799Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:55.06275848Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_internal_dns_ced4c03d-9859-49c7-a5b2-d6706c87f34e"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:55.062807831Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_internal_dns_21e40459-2dec-4da5-8537-0450d95100c7"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:55.062814151Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_internal_dns_c9fa5d8b-74a4-4a22-951f-5a84f02a69e3"} {"msg":"Writing ledger to /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:55.062825101Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:55.063145476Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:55.063444061Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"c9de9ac4-67d8-4cbf-aeef-f6ace7a756a3","remote_addr":"[fd00:1122:3344:101::1]:50004","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":8304832,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:55.065372773Z","hostname":"buskin","pid":1562,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:555"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:55.120158248Z","hostname":"buskin","pid":1562,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:555"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:55.172232259Z","hostname":"buskin","pid":1562,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:555"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:55.220871244Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:591"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:55.220882365Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:260"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:55.253515298Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:55.255061513Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:302"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:55.255074963Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21e40459-2dec-4da5-8537-0450d95100c7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 27200e24-efc4-4a6c-b7d3-7afde4232881, kind: Service { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a }, name: Name(\"ntp-3f5202b7-215d-4f5c-8883-5d9b359fe25a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 2])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:55.285536271Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:55.285717154Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:62076"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:55.285931328Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:55.285937418Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:55.306007576Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:55.306053596Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:55.322597137Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:55.322637567Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:55.337660753Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:55.337710814Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:55.337882817Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:55.337974298Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:55.348713883Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:260"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:55.381895206Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:55.382675009Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:302"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:55.382680109Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:55.412535407Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:424","port":"Port { inner: PortInner(PortData { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 195, 2]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:55.412550237Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1235","zone_type":"boundary_ntp"} {"msg":"no nat entry found for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:55.412789351Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:55.412794721Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:55.413016405Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:55.413501092Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Configuring new Omicron zone: oxz_ntp_3f5202b7-215d-4f5c-8883-5d9b359fe25a","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:55.437623827Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_ntp_3f5202b7-215d-4f5c-8883-5d9b359fe25a","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:55.454858328Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_ntp_3f5202b7-215d-4f5c-8883-5d9b359fe25a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:56.070565911Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_ntp_3f5202b7-215d-4f5c-8883-5d9b359fe25a zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:45:56.070736333Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_3f5202b7-215d-4f5c-8883-5d9b359fe25a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_ntp_3f5202b7-215d-4f5c-8883-5d9b359fe25a\") failed: Property not found. retry in 28.282024ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:56.197007817Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_3f5202b7-215d-4f5c-8883-5d9b359fe25a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_3f5202b7-215d-4f5c-8883-5d9b359fe25a\") failed: Property not found. retry in 82.426945ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:56.233058456Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_3f5202b7-215d-4f5c-8883-5d9b359fe25a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_3f5202b7-215d-4f5c-8883-5d9b359fe25a\") failed: Property not found. retry in 154.568791ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:56.327125764Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_3f5202b7-215d-4f5c-8883-5d9b359fe25a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_3f5202b7-215d-4f5c-8883-5d9b359fe25a\") failed: Property not found. retry in 497.97113ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:56.494347457Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_3f5202b7-215d-4f5c-8883-5d9b359fe25a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_3f5202b7-215d-4f5c-8883-5d9b359fe25a\") failed: Property not found. retry in 1.069646889s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:57.002360978Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_3f5202b7-215d-4f5c-8883-5d9b359fe25a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_3f5202b7-215d-4f5c-8883-5d9b359fe25a\") failed: Property not found. retry in 691.246729ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:58.082167104Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_3f5202b7-215d-4f5c-8883-5d9b359fe25a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_3f5202b7-215d-4f5c-8883-5d9b359fe25a\") failed: Property not found. retry in 685.725816ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:58.782353747Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_3f5202b7-215d-4f5c-8883-5d9b359fe25a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_3f5202b7-215d-4f5c-8883-5d9b359fe25a\") failed: Property not found. retry in 847.972626ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:45:59.47646903Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_3f5202b7-215d-4f5c-8883-5d9b359fe25a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_3f5202b7-215d-4f5c-8883-5d9b359fe25a\") failed: Property not found. retry in 573.937948ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:00.334518013Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_3f5202b7-215d-4f5c-8883-5d9b359fe25a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_3f5202b7-215d-4f5c-8883-5d9b359fe25a\") failed: Property not found. retry in 852.195636ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:00.918479606Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_3f5202b7-215d-4f5c-8883-5d9b359fe25a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:01.791356971Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_ntp_3f5202b7-215d-4f5c-8883-5d9b359fe25a"} {"msg":"Writing ledger to /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:01.791425392Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:01.791758637Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:01.792095933Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"cde3975d-6a99-420c-80d1-5da023b940b3","remote_addr":"[fd00:1122:3344:101::1]:62076","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":6506325,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:01.794334429Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:01.824659755Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:01.824799197Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:40453"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_3f5202b7-215d-4f5c-8883-5d9b359fe25a': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-08-27T19:46:01.832368251Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3847"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:01.832480433Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"7aa27276-fb15-43d1-ae49-e39ca1b1f104","remote_addr":"[fd00:1122:3344:101::1]:40453","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":7560,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:01.834579577Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:662","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:02.013368189Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:02.013498791Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:41616"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_3f5202b7-215d-4f5c-8883-5d9b359fe25a': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-08-27T19:46:06.023138477Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3847"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:06.023264159Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"60a64cb6-699d-4be7-8224-d0c293d8aa43","remote_addr":"[fd00:1122:3344:101::1]:41616","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4009664,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:06.026773566Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:662","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:06.169327886Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:06.169472738Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:47111"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:06.178795831Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"8e0fc093-d92b-4d84-be38-2718e510e237","remote_addr":"[fd00:1122:3344:101::1]:47111","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":9267,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:06.180744253Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:640"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:06.180750222Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:662","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:06.318921761Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:06.319046683Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:65386"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:06.326884461Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"7ded7aa2-e365-4ea7-8765-fd697fa85627","remote_addr":"[fd00:1122:3344:101::1]:65386","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":7799,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:06.329509954Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:640"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:06.329517174Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:662","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:06.585718401Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:06.585830693Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:45974"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:06.593260804Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"ff28ea76-3171-4a25-ab07-9b7f143350a3","remote_addr":"[fd00:1122:3344:101::1]:45974","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":7404,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:06.596160802Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:640"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:06.596168362Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:662","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:06.798698242Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:06.798823654Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:55563"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:06.805866939Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"1a642c2e-5f88-4537-b52b-a59175553fed","remote_addr":"[fd00:1122:3344:101::1]:55563","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":7015,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:06.80900949Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:640"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:06.80902825Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:662","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:07.125445492Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:07.125550683Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:45945"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:07.133089846Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"eb3f89b7-7300-42c2-a2fe-c7cbae8f0afa","remote_addr":"[fd00:1122:3344:101::1]:45945","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":7516,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:07.135936783Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:640"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:07.135943773Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:662","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:07.491536895Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:07.491646176Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:42369"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:07.499377593Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"9c833322-e79a-4279-a552-9fcabcb99c13","remote_addr":"[fd00:1122:3344:101::1]:42369","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":7705,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:07.502199199Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:640"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:07.502206149Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:662","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:08.250175867Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:08.250301579Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:46315"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:08.269121117Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"6cfa45e5-a9ad-4261-a487-bbcb890e47a3","remote_addr":"[fd00:1122:3344:101::1]:46315","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":18742,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 3489030767, ip_addr: 207.246.98.111, stratum: 3, ref_time: 1724787968.0499272, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:08.272055385Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:640"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:08.272062285Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1266"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21e40459-2dec-4da5-8537-0450d95100c7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 27200e24-efc4-4a6c-b7d3-7afde4232881, kind: Service { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a }, name: Name(\"ntp-3f5202b7-215d-4f5c-8883-5d9b359fe25a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 2])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: edf311b1-5e51-4039-b1b2-78828b2fe27a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f7611b-d552-42d2-b5e6-6a3218324db7, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } } } }, OmicronZoneConfig { id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:08.303355806Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:08.30353566Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:36351"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:08.303712802Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:08.303718483Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:08.324095285Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:08.324121886Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:08.343557663Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:08.343593654Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:08.362033576Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:08.362060746Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:08.415806564Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:08.415836705Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.024133585Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.024162085Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.078149017Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.078177778Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.049798064Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.049827175Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.106798806Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.106834946Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.107004109Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.107102631Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:09.139624762Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:09.139671303Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:09.139679233Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:09.139687753Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:09.139694393Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.139786935Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"b29c4877-7cd3-4394-b761-05c7a29725cd","remote_addr":"[fd00:1122:3344:101::1]:36351","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634\", TimeNotSynchronized), (\"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14\", TimeNotSynchronized), (\"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6\", TimeNotSynchronized), (\"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a\", TimeNotSynchronized), (\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\", TimeNotSynchronized)]","latency_us":362191,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:09.139963377Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"55.352801ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b29c4877-7cd3-4394-b761-05c7a29725cd\", \"content-length\": \"124\", \"date\": \"Tue, 27 Aug 2024 19:46:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b29c4877-7cd3-4394-b761-05c7a29725cd\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21e40459-2dec-4da5-8537-0450d95100c7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 27200e24-efc4-4a6c-b7d3-7afde4232881, kind: Service { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a }, name: Name(\"ntp-3f5202b7-215d-4f5c-8883-5d9b359fe25a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 2])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: edf311b1-5e51-4039-b1b2-78828b2fe27a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f7611b-d552-42d2-b5e6-6a3218324db7, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } } } }, OmicronZoneConfig { id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.196136996Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.196518462Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.196526652Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.215623154Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.215664665Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.233253752Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.233290313Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.251027943Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.251062503Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.269378303Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.269411303Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.288345373Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.288381813Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.305736257Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.305769147Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.323552188Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.323591349Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.341513971Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.341563802Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.341773956Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.341931658Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:09.373676437Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:09.373705028Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:09.373713808Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:09.373720378Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:09.373728628Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.37384634Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"4c04df33-2188-4c61-9897-82477933903b","remote_addr":"[fd00:1122:3344:101::1]:36351","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6\", TimeNotSynchronized), (\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\", TimeNotSynchronized), (\"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634\", TimeNotSynchronized), (\"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a\", TimeNotSynchronized), (\"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14\", TimeNotSynchronized)]","latency_us":177532,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:09.373994052Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"149.164248ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4c04df33-2188-4c61-9897-82477933903b\", \"content-length\": \"124\", \"date\": \"Tue, 27 Aug 2024 19:46:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4c04df33-2188-4c61-9897-82477933903b\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21e40459-2dec-4da5-8537-0450d95100c7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 27200e24-efc4-4a6c-b7d3-7afde4232881, kind: Service { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a }, name: Name(\"ntp-3f5202b7-215d-4f5c-8883-5d9b359fe25a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 2])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: edf311b1-5e51-4039-b1b2-78828b2fe27a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f7611b-d552-42d2-b5e6-6a3218324db7, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } } } }, OmicronZoneConfig { id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.52436021Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.524711125Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.524720126Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.542810621Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.542858432Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.560436629Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.56047264Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.578073077Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.578107688Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.595919519Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.59595486Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.613532867Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.613572938Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.631305098Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.631340938Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.648993687Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.649022087Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.667049622Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.667077862Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.667249125Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.667357977Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:09.698266372Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:09.698303632Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:09.698311002Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:09.698316302Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:09.698321353Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.698411744Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"7a467dc0-0edf-46a7-a37b-d3def7373dc3","remote_addr":"[fd00:1122:3344:101::1]:36351","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6\", TimeNotSynchronized), (\"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a\", TimeNotSynchronized), (\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\", TimeNotSynchronized), (\"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14\", TimeNotSynchronized), (\"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634\", TimeNotSynchronized)]","latency_us":173883,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:09.698556646Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"99.218081ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7a467dc0-0edf-46a7-a37b-d3def7373dc3\", \"content-length\": \"124\", \"date\": \"Tue, 27 Aug 2024 19:46:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7a467dc0-0edf-46a7-a37b-d3def7373dc3\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21e40459-2dec-4da5-8537-0450d95100c7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 27200e24-efc4-4a6c-b7d3-7afde4232881, kind: Service { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a }, name: Name(\"ntp-3f5202b7-215d-4f5c-8883-5d9b359fe25a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 2])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: edf311b1-5e51-4039-b1b2-78828b2fe27a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f7611b-d552-42d2-b5e6-6a3218324db7, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } } } }, OmicronZoneConfig { id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.798915987Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.799290953Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.799301113Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.817023863Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.817053553Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.83459719Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.83463239Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.852601974Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.852631215Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.871967931Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.872001921Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.889729871Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.889758111Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.907258807Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.907286858Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.924355917Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.924382337Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.944605728Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.944639598Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.944796021Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.944897203Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:09.978177306Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:09.978209967Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:09.978217207Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:09.978223057Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:09.978230227Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:09.978326109Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"cba21be4-eb13-47b3-b149-886968604abb","remote_addr":"[fd00:1122:3344:101::1]:36351","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a\", TimeNotSynchronized), (\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\", TimeNotSynchronized), (\"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14\", TimeNotSynchronized), (\"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6\", TimeNotSynchronized), (\"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634\", TimeNotSynchronized)]","latency_us":179232,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:09.978475641Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"114.921707ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"cba21be4-eb13-47b3-b149-886968604abb\", \"content-length\": \"124\", \"date\": \"Tue, 27 Aug 2024 19:46:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"cba21be4-eb13-47b3-b149-886968604abb\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21e40459-2dec-4da5-8537-0450d95100c7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 27200e24-efc4-4a6c-b7d3-7afde4232881, kind: Service { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a }, name: Name(\"ntp-3f5202b7-215d-4f5c-8883-5d9b359fe25a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 2])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: edf311b1-5e51-4039-b1b2-78828b2fe27a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f7611b-d552-42d2-b5e6-6a3218324db7, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } } } }, OmicronZoneConfig { id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:10.094851813Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:10.09526919Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:10.09528107Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:10.114777039Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:10.114816729Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:10.13196962Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:10.13199531Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:10.150867649Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:10.15091839Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:10.166432273Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:10.166462004Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:10.182907162Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:10.182942443Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:10.202168947Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:10.202198017Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:10.219061133Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:10.219092674Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:10.236981286Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:10.237026237Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:10.23723886Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:10.237336102Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:10.267602376Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:10.267631547Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:10.267638007Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:10.267656067Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:10.267662567Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:10.267729559Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"0070bfb3-b09e-45f8-8755-e681b4c79f9a","remote_addr":"[fd00:1122:3344:101::1]:36351","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6\", TimeNotSynchronized), (\"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14\", TimeNotSynchronized), (\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\", TimeNotSynchronized), (\"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634\", TimeNotSynchronized), (\"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a\", TimeNotSynchronized)]","latency_us":172684,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:10.267849761Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"303.117612ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0070bfb3-b09e-45f8-8755-e681b4c79f9a\", \"content-length\": \"124\", \"date\": \"Tue, 27 Aug 2024 19:46:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0070bfb3-b09e-45f8-8755-e681b4c79f9a\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21e40459-2dec-4da5-8537-0450d95100c7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 27200e24-efc4-4a6c-b7d3-7afde4232881, kind: Service { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a }, name: Name(\"ntp-3f5202b7-215d-4f5c-8883-5d9b359fe25a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 2])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: edf311b1-5e51-4039-b1b2-78828b2fe27a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f7611b-d552-42d2-b5e6-6a3218324db7, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } } } }, OmicronZoneConfig { id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:10.572226705Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:10.572413758Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:10.572418858Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:10.591099303Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:10.591135084Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:10.608497278Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:10.608535738Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:10.626170897Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:10.626199977Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:10.64412583Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:10.64415269Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:10.661684757Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:10.661713167Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:10.679140572Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:10.679173433Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:10.698240353Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:10.698269094Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:10.717367486Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:10.717398187Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:10.717579899Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:10.717678561Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:10.749502411Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:10.749555722Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:10.749565872Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:10.749575632Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:10.749583382Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:10.749701344Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"111f6e00-5484-45fe-b70b-42737d71af6f","remote_addr":"[fd00:1122:3344:101::1]:36351","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14\", TimeNotSynchronized), (\"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634\", TimeNotSynchronized), (\"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a\", TimeNotSynchronized), (\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\", TimeNotSynchronized), (\"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6\", TimeNotSynchronized)]","latency_us":177371,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:10.749893587Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"240.298548ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"111f6e00-5484-45fe-b70b-42737d71af6f\", \"content-length\": \"124\", \"date\": \"Tue, 27 Aug 2024 19:46:10 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"111f6e00-5484-45fe-b70b-42737d71af6f\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21e40459-2dec-4da5-8537-0450d95100c7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 27200e24-efc4-4a6c-b7d3-7afde4232881, kind: Service { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a }, name: Name(\"ntp-3f5202b7-215d-4f5c-8883-5d9b359fe25a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 2])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: edf311b1-5e51-4039-b1b2-78828b2fe27a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f7611b-d552-42d2-b5e6-6a3218324db7, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } } } }, OmicronZoneConfig { id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:10.991313753Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:10.991576247Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:10.991585097Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:11.011511803Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:11.011558924Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:11.031038414Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:11.031077934Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:11.049510196Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:11.049544817Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:11.067847321Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:11.067871687Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:11.085412085Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:11.085441785Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:11.10402982Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:11.10406199Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:11.121832902Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:11.121858342Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:11.139485221Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:11.139560342Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:11.139731135Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:11.139831056Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:11.172193527Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:11.172220507Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:11.172226867Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:11.172231887Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:11.172238168Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:11.172328169Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"5ebeba51-9c29-4506-8a5c-28798e0792da","remote_addr":"[fd00:1122:3344:101::1]:36351","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14\", TimeNotSynchronized), (\"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a\", TimeNotSynchronized), (\"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6\", TimeNotSynchronized), (\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\", TimeNotSynchronized), (\"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634\", TimeNotSynchronized)]","latency_us":181348,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:11.172470301Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"305.324522ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5ebeba51-9c29-4506-8a5c-28798e0792da\", \"content-length\": \"124\", \"date\": \"Tue, 27 Aug 2024 19:46:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5ebeba51-9c29-4506-8a5c-28798e0792da\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21e40459-2dec-4da5-8537-0450d95100c7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 27200e24-efc4-4a6c-b7d3-7afde4232881, kind: Service { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a }, name: Name(\"ntp-3f5202b7-215d-4f5c-8883-5d9b359fe25a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 2])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: edf311b1-5e51-4039-b1b2-78828b2fe27a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f7611b-d552-42d2-b5e6-6a3218324db7, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } } } }, OmicronZoneConfig { id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:11.477841765Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:11.478163081Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:11.478176011Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:11.496456581Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:11.496499771Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:11.514046049Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:11.514068179Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:11.530592091Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:11.53063019Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:11.548535514Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:11.548558844Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:11.56597793Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:11.56600084Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:11.583003719Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:11.583029669Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:11.599884245Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:11.599907336Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:11.617475464Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:11.617506934Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:11.617686627Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:11.617789869Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:11.65022725Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:11.650281661Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:11.650299302Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:11.650306782Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:11.650315232Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:11.650391063Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"29e99086-421a-4e7d-98c8-7f24ee441174","remote_addr":"[fd00:1122:3344:101::1]:36351","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634\", TimeNotSynchronized), (\"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a\", TimeNotSynchronized), (\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\", TimeNotSynchronized), (\"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14\", TimeNotSynchronized), (\"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6\", TimeNotSynchronized)]","latency_us":172877,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:11.650530015Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"462.500952ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"29e99086-421a-4e7d-98c8-7f24ee441174\", \"content-length\": \"124\", \"date\": \"Tue, 27 Aug 2024 19:46:10 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"29e99086-421a-4e7d-98c8-7f24ee441174\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21e40459-2dec-4da5-8537-0450d95100c7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 27200e24-efc4-4a6c-b7d3-7afde4232881, kind: Service { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a }, name: Name(\"ntp-3f5202b7-215d-4f5c-8883-5d9b359fe25a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 2])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: edf311b1-5e51-4039-b1b2-78828b2fe27a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f7611b-d552-42d2-b5e6-6a3218324db7, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } } } }, OmicronZoneConfig { id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:12.113620953Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:12.113824017Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:12.113834437Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:12.132938112Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:12.13297246Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:12.152048023Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:12.152079994Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:12.169749493Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:12.169784434Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:12.18843842Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:12.18847115Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:12.206272982Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:12.206316282Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:12.225374715Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:12.225403475Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:12.242838561Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:12.242869692Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:12.261180752Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:12.261218232Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:12.261415175Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:12.261524467Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:12.293025013Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:12.293061994Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:12.293070694Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:12.293077594Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:12.293086054Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:12.293193916Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"6684b4c7-5e44-463b-aa3d-788dfa21ccc0","remote_addr":"[fd00:1122:3344:101::1]:36351","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a\", TimeNotSynchronized), (\"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14\", TimeNotSynchronized), (\"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6\", TimeNotSynchronized), (\"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634\", TimeNotSynchronized), (\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\", TimeNotSynchronized)]","latency_us":179964,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:12.293367549Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"427.7105ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6684b4c7-5e44-463b-aa3d-788dfa21ccc0\", \"content-length\": \"124\", \"date\": \"Tue, 27 Aug 2024 19:46:12 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6684b4c7-5e44-463b-aa3d-788dfa21ccc0\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21e40459-2dec-4da5-8537-0450d95100c7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 27200e24-efc4-4a6c-b7d3-7afde4232881, kind: Service { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a }, name: Name(\"ntp-3f5202b7-215d-4f5c-8883-5d9b359fe25a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 2])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: edf311b1-5e51-4039-b1b2-78828b2fe27a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f7611b-d552-42d2-b5e6-6a3218324db7, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } } } }, OmicronZoneConfig { id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:12.720887185Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:12.72119455Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:12.72120931Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:12.738601745Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:12.738648206Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:12.756509778Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:12.756552229Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:12.775006162Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:12.775048312Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:12.792879925Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:12.792916955Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:12.809816332Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:12.809861783Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:12.828352356Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:12.828391696Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:12.846147338Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:12.846184978Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:12.863588563Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:12.863632314Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:12.863844297Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:12.863976467Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:12.89571738Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:12.895776231Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:12.895786501Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:12.895796131Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:12.895807821Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:12.895905423Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"82a2fd6d-e9db-43ce-bb53-a486a5c424d1","remote_addr":"[fd00:1122:3344:101::1]:36351","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a\", TimeNotSynchronized), (\"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14\", TimeNotSynchronized), (\"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6\", TimeNotSynchronized), (\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\", TimeNotSynchronized), (\"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634\", TimeNotSynchronized)]","latency_us":175342,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:12.896039835Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"316.048976ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"82a2fd6d-e9db-43ce-bb53-a486a5c424d1\", \"content-length\": \"124\", \"date\": \"Tue, 27 Aug 2024 19:46:12 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"82a2fd6d-e9db-43ce-bb53-a486a5c424d1\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21e40459-2dec-4da5-8537-0450d95100c7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 27200e24-efc4-4a6c-b7d3-7afde4232881, kind: Service { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a }, name: Name(\"ntp-3f5202b7-215d-4f5c-8883-5d9b359fe25a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 2])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: edf311b1-5e51-4039-b1b2-78828b2fe27a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f7611b-d552-42d2-b5e6-6a3218324db7, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } } } }, OmicronZoneConfig { id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:13.212626263Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:13.212949728Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:13.212960298Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:13.232818774Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:13.232870285Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:13.251916329Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:13.251949377Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:13.269070188Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:13.269104509Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:13.287754174Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:13.287790515Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:13.30640218Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:13.30643515Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:13.323422039Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:13.323459029Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:13.341146769Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:13.34119507Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:13.35886444Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:13.35890372Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:13.359110784Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:13.359242266Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:13.391460714Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:13.391506424Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:13.391515039Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:13.391522885Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:13.391539873Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:13.391611886Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"063ca948-7c38-436f-ad1d-665952527dbc","remote_addr":"[fd00:1122:3344:101::1]:36351","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634\", TimeNotSynchronized), (\"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6\", TimeNotSynchronized), (\"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14\", TimeNotSynchronized), (\"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a\", TimeNotSynchronized), (\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\", TimeNotSynchronized)]","latency_us":179340,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:13.391739718Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"813.230539ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"063ca948-7c38-436f-ad1d-665952527dbc\", \"content-length\": \"124\", \"date\": \"Tue, 27 Aug 2024 19:46:13 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"063ca948-7c38-436f-ad1d-665952527dbc\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21e40459-2dec-4da5-8537-0450d95100c7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 27200e24-efc4-4a6c-b7d3-7afde4232881, kind: Service { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a }, name: Name(\"ntp-3f5202b7-215d-4f5c-8883-5d9b359fe25a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 2])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: edf311b1-5e51-4039-b1b2-78828b2fe27a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f7611b-d552-42d2-b5e6-6a3218324db7, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } } } }, OmicronZoneConfig { id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:14.204055599Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:14.204435485Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:14.204453085Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:14.224648407Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:14.224689517Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:14.242947456Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:14.242983457Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:14.26025319Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:14.260286871Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:14.278140748Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:14.278174584Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:14.295893064Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:14.295931335Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:14.3139721Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:14.314026711Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:14.332648766Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:14.332687587Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:14.350827504Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:14.350881595Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:14.351066018Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:14.35117332Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:14.382560954Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:14.382598775Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:14.382605915Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:14.382614915Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:14.382623305Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:14.382742567Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"1a272a49-c91d-4553-b27a-11056ebd2d14","remote_addr":"[fd00:1122:3344:101::1]:36351","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a\", TimeNotSynchronized), (\"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634\", TimeNotSynchronized), (\"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14\", TimeNotSynchronized), (\"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6\", TimeNotSynchronized), (\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\", TimeNotSynchronized)]","latency_us":179013,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:14.38289399Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"551.921538ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1a272a49-c91d-4553-b27a-11056ebd2d14\", \"content-length\": \"124\", \"date\": \"Tue, 27 Aug 2024 19:46:14 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1a272a49-c91d-4553-b27a-11056ebd2d14\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21e40459-2dec-4da5-8537-0450d95100c7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 27200e24-efc4-4a6c-b7d3-7afde4232881, kind: Service { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a }, name: Name(\"ntp-3f5202b7-215d-4f5c-8883-5d9b359fe25a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 2])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: edf311b1-5e51-4039-b1b2-78828b2fe27a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f7611b-d552-42d2-b5e6-6a3218324db7, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } } } }, OmicronZoneConfig { id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:14.934856064Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:14.9352293Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:14.93524274Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:14.954870082Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:14.954912993Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:14.972611973Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:14.972653873Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:14.990191735Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:14.990234392Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:15.008130313Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:15.008158475Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:15.027705066Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:15.027742336Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:15.047022913Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:15.047063303Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:15.064739193Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:15.064789834Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:15.082633316Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:15.082670607Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:15.082905995Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:15.083028623Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:15.115444084Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:15.115472245Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:15.115478655Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:15.115483675Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:15.115488725Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:15.115602597Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"9b4e45e1-6a3b-431c-89e9-0365f5b6708d","remote_addr":"[fd00:1122:3344:101::1]:36351","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a\", TimeNotSynchronized), (\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\", TimeNotSynchronized), (\"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6\", TimeNotSynchronized), (\"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634\", TimeNotSynchronized), (\"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14\", TimeNotSynchronized)]","latency_us":181119,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:15.115762369Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"938.577965ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9b4e45e1-6a3b-431c-89e9-0365f5b6708d\", \"content-length\": \"124\", \"date\": \"Tue, 27 Aug 2024 19:46:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9b4e45e1-6a3b-431c-89e9-0365f5b6708d\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21e40459-2dec-4da5-8537-0450d95100c7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 27200e24-efc4-4a6c-b7d3-7afde4232881, kind: Service { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a }, name: Name(\"ntp-3f5202b7-215d-4f5c-8883-5d9b359fe25a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 2])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: edf311b1-5e51-4039-b1b2-78828b2fe27a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f7611b-d552-42d2-b5e6-6a3218324db7, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } } } }, OmicronZoneConfig { id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:16.05210765Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:16.052471866Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:16.052486136Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:16.073780865Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:16.073817566Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:16.092208447Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:16.092242538Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:16.110192292Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:16.110223483Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:16.127877032Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:16.127900783Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:16.145688244Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:16.145715624Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:16.163206201Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:16.163229612Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:16.181124805Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:16.181148605Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:16.198607562Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:16.198640292Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:16.198835585Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:16.198943397Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:16.231221706Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:16.231264691Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:16.231271591Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:16.231279391Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:16.231285637Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:16.231368748Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"28e0be4b-2816-4bc2-a1bd-0973cb3b5111","remote_addr":"[fd00:1122:3344:101::1]:36351","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a\", TimeNotSynchronized), (\"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6\", TimeNotSynchronized), (\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\", TimeNotSynchronized), (\"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14\", TimeNotSynchronized), (\"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634\", TimeNotSynchronized)]","latency_us":179634,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:16.231516021Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"776.928213ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"28e0be4b-2816-4bc2-a1bd-0973cb3b5111\", \"content-length\": \"124\", \"date\": \"Tue, 27 Aug 2024 19:46:16 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"28e0be4b-2816-4bc2-a1bd-0973cb3b5111\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21e40459-2dec-4da5-8537-0450d95100c7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 27200e24-efc4-4a6c-b7d3-7afde4232881, kind: Service { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a }, name: Name(\"ntp-3f5202b7-215d-4f5c-8883-5d9b359fe25a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 2])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: edf311b1-5e51-4039-b1b2-78828b2fe27a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f7611b-d552-42d2-b5e6-6a3218324db7, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } } } }, OmicronZoneConfig { id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:17.007009761Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:17.00817624Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:17.008184931Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:17.028371751Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:17.028409942Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:17.048653414Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:17.048700075Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:17.068430478Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:17.068469228Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:17.087291517Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:17.087338788Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:17.105001407Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:17.105049468Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:17.122594895Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:17.122641345Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:17.141305522Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:17.141337913Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:17.159133344Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:17.159173085Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:17.159388838Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:17.159529557Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:17.192902498Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:17.192942428Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:17.192950528Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:17.192956668Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:17.192963999Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:17.19307319Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"6a3af777-fb46-44aa-a885-a7a015f361c8","remote_addr":"[fd00:1122:3344:101::1]:36351","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634\", TimeNotSynchronized), (\"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6\", TimeNotSynchronized), (\"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a\", TimeNotSynchronized), (\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\", TimeNotSynchronized), (\"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14\", TimeNotSynchronized)]","latency_us":186406,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:17.193239433Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"1.170502365s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6a3af777-fb46-44aa-a885-a7a015f361c8\", \"content-length\": \"124\", \"date\": \"Tue, 27 Aug 2024 19:46:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6a3af777-fb46-44aa-a885-a7a015f361c8\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21e40459-2dec-4da5-8537-0450d95100c7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 27200e24-efc4-4a6c-b7d3-7afde4232881, kind: Service { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a }, name: Name(\"ntp-3f5202b7-215d-4f5c-8883-5d9b359fe25a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 2])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: edf311b1-5e51-4039-b1b2-78828b2fe27a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f7611b-d552-42d2-b5e6-6a3218324db7, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } } } }, OmicronZoneConfig { id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:18.361497797Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:18.361830842Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:18.361852133Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:18.381602226Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:18.381638397Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:18.401479542Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:18.401514113Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:18.420332561Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:18.420378822Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:18.438294895Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:18.438329656Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:18.457022737Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:18.457056643Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:18.475945743Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:18.47598478Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:18.494648759Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:18.49468856Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:18.51301826Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:18.513060591Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:18.513210393Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:18.513298705Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:18.545456862Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:18.545512723Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:18.545546403Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:18.545557393Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:18.545567563Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:18.545680205Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"06f96e18-2565-438c-8e2c-abb1fc935b42","remote_addr":"[fd00:1122:3344:101::1]:36351","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634\", TimeNotSynchronized), (\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\", TimeNotSynchronized), (\"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a\", TimeNotSynchronized), (\"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6\", TimeNotSynchronized), (\"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14\", TimeNotSynchronized)]","latency_us":184530,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:18.545869888Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"816.107674ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"06f96e18-2565-438c-8e2c-abb1fc935b42\", \"content-length\": \"124\", \"date\": \"Tue, 27 Aug 2024 19:46:18 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"06f96e18-2565-438c-8e2c-abb1fc935b42\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21e40459-2dec-4da5-8537-0450d95100c7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 27200e24-efc4-4a6c-b7d3-7afde4232881, kind: Service { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a }, name: Name(\"ntp-3f5202b7-215d-4f5c-8883-5d9b359fe25a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 2])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: edf311b1-5e51-4039-b1b2-78828b2fe27a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f7611b-d552-42d2-b5e6-6a3218324db7, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } } } }, OmicronZoneConfig { id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:19.361251859Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:19.361573662Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:19.361610855Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:19.380441444Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:19.380495915Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:19.400279609Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:19.400311949Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:19.417493961Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:19.417529621Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:19.435910173Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:19.435945573Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:19.455113607Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:19.455149118Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:19.47235799Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:19.47238848Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:19.491432652Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:19.491464623Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:19.508866048Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:19.508910589Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:19.509118722Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:19.509242635Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:19.540858573Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:19.540896583Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:19.540905423Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:19.540913583Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:19.540920254Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:19.541025555Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"2551d53d-182a-47d7-a411-45a674823b31","remote_addr":"[fd00:1122:3344:101::1]:36351","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a\", TimeNotSynchronized), (\"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6\", TimeNotSynchronized), (\"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634\", TimeNotSynchronized), (\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\", TimeNotSynchronized), (\"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14\", TimeNotSynchronized)]","latency_us":180105,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:19.541208128Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"2.102067967s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2551d53d-182a-47d7-a411-45a674823b31\", \"content-length\": \"124\", \"date\": \"Tue, 27 Aug 2024 19:46:19 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2551d53d-182a-47d7-a411-45a674823b31\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21e40459-2dec-4da5-8537-0450d95100c7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 27200e24-efc4-4a6c-b7d3-7afde4232881, kind: Service { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a }, name: Name(\"ntp-3f5202b7-215d-4f5c-8883-5d9b359fe25a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 2])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: edf311b1-5e51-4039-b1b2-78828b2fe27a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f7611b-d552-42d2-b5e6-6a3218324db7, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } } } }, OmicronZoneConfig { id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:21.638356913Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:21.638733169Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:21.638748929Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:21.658330297Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:21.658362717Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:21.677703528Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:21.677728978Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:21.696590457Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:21.696621298Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:21.7156959Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:21.715721591Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:21.733299709Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:21.733336099Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:21.751579538Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:21.751611079Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:21.764943397Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"230.697487ms"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:21.769772066Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:21.769779916Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:21.787752411Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:21.787782651Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:21.787975414Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:21.788096217Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:21.81941748Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:21.81946512Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:21.819474921Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:21.819482261Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:21.819489281Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:21.819578282Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"a3afd711-6609-4764-9a9f-bd0283b16182","remote_addr":"[fd00:1122:3344:101::1]:36351","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14\", TimeNotSynchronized), (\"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a\", TimeNotSynchronized), (\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\", TimeNotSynchronized), (\"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634\", TimeNotSynchronized), (\"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6\", TimeNotSynchronized)]","latency_us":181553,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:21.819734495Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"2.324152973s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a3afd711-6609-4764-9a9f-bd0283b16182\", \"content-length\": \"124\", \"date\": \"Tue, 27 Aug 2024 19:46:21 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a3afd711-6609-4764-9a9f-bd0283b16182\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:23.760603689Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63016"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:23.760785603Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"bef0ca93-cf12-4226-ab87-0bcd65c7a521","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63016","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":81,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21e40459-2dec-4da5-8537-0450d95100c7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 27200e24-efc4-4a6c-b7d3-7afde4232881, kind: Service { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a }, name: Name(\"ntp-3f5202b7-215d-4f5c-8883-5d9b359fe25a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 2])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: edf311b1-5e51-4039-b1b2-78828b2fe27a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f7611b-d552-42d2-b5e6-6a3218324db7, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } } } }, OmicronZoneConfig { id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:24.139348276Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:24.139699652Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:24.139714182Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:24.161015221Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:24.161070612Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:24.179516666Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:24.179554275Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:24.197317586Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:24.197359627Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:24.216510101Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:24.216543551Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:24.235344616Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:24.23538787Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:24.254142927Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:24.254179828Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:24.273047787Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:24.273081108Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:24.291809595Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:24.291835585Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:24.292046549Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:24.292187251Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:24.323136858Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:24.323171888Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:24.323179879Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:24.323186749Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:24.323194709Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:24.323299831Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"de3e3660-31ef-4f04-a305-4006dc970f18","remote_addr":"[fd00:1122:3344:101::1]:36351","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14\", TimeNotSynchronized), (\"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a\", TimeNotSynchronized), (\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\", TimeNotSynchronized), (\"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6\", TimeNotSynchronized), (\"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634\", TimeNotSynchronized)]","latency_us":184309,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:24.323453573Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"1.464650859s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"de3e3660-31ef-4f04-a305-4006dc970f18\", \"content-length\": \"124\", \"date\": \"Tue, 27 Aug 2024 19:46:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"de3e3660-31ef-4f04-a305-4006dc970f18\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21e40459-2dec-4da5-8537-0450d95100c7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 27200e24-efc4-4a6c-b7d3-7afde4232881, kind: Service { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a }, name: Name(\"ntp-3f5202b7-215d-4f5c-8883-5d9b359fe25a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 2])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: edf311b1-5e51-4039-b1b2-78828b2fe27a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f7611b-d552-42d2-b5e6-6a3218324db7, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } } } }, OmicronZoneConfig { id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:25.785296887Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:25.786379115Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:25.786390125Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:25.806552315Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:25.806586206Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:25.8257465Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:25.82578579Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:25.842918121Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:25.842956332Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:25.861425005Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:25.861458365Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:25.880750931Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:25.880786962Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:25.897880452Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:25.897913732Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:25.917114567Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:25.917165108Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:25.934159146Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:25.934200867Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:25.93436194Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:25.934465391Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:25.966379224Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:25.966426205Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:25.966434555Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:25.966441135Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:25.966447355Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:25.966557967Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"7d0ad4d2-f2f6-4d85-9b13-48fda994309c","remote_addr":"[fd00:1122:3344:101::1]:36351","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14\", TimeNotSynchronized), (\"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a\", TimeNotSynchronized), (\"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6\", TimeNotSynchronized), (\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\", TimeNotSynchronized), (\"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634\", TimeNotSynchronized)]","latency_us":181582,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:25.966768501Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"2.641549568s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7d0ad4d2-f2f6-4d85-9b13-48fda994309c\", \"content-length\": \"124\", \"date\": \"Tue, 27 Aug 2024 19:46:25 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7d0ad4d2-f2f6-4d85-9b13-48fda994309c\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21e40459-2dec-4da5-8537-0450d95100c7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 27200e24-efc4-4a6c-b7d3-7afde4232881, kind: Service { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a }, name: Name(\"ntp-3f5202b7-215d-4f5c-8883-5d9b359fe25a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 2])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: edf311b1-5e51-4039-b1b2-78828b2fe27a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f7611b-d552-42d2-b5e6-6a3218324db7, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } } } }, OmicronZoneConfig { id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:28.602308519Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:28.60358582Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:28.60359222Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:28.625065721Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:28.625107912Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:28.644361378Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:28.644403158Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:28.662599526Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:28.662644127Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:28.680847335Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:28.680880815Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:28.699133445Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:28.699176585Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:28.717522486Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:28.717561256Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:28.735701634Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:28.735744865Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:28.754323039Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:28.75437433Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:28.754577423Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:28.754689575Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:28.787698776Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:28.787743847Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:28.787753107Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:28.787759717Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:28.787766027Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:28.787868409Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"c7e9ccfa-b2fb-42f5-a1f8-b3972355c1c5","remote_addr":"[fd00:1122:3344:101::1]:36351","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\", TimeNotSynchronized), (\"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14\", TimeNotSynchronized), (\"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634\", TimeNotSynchronized), (\"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a\", TimeNotSynchronized), (\"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6\", TimeNotSynchronized)]","latency_us":185893,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:28.788042323Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"2.805511648s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c7e9ccfa-b2fb-42f5-a1f8-b3972355c1c5\", \"content-length\": \"124\", \"date\": \"Tue, 27 Aug 2024 19:46:28 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c7e9ccfa-b2fb-42f5-a1f8-b3972355c1c5\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21e40459-2dec-4da5-8537-0450d95100c7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 27200e24-efc4-4a6c-b7d3-7afde4232881, kind: Service { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a }, name: Name(\"ntp-3f5202b7-215d-4f5c-8883-5d9b359fe25a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 2])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: edf311b1-5e51-4039-b1b2-78828b2fe27a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f7611b-d552-42d2-b5e6-6a3218324db7, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } } } }, OmicronZoneConfig { id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:31.586440558Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:31.587631118Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:31.587640148Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:31.60791557Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:31.607948071Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:31.626997623Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:31.627021753Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:31.644649651Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:31.644675332Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:31.662656676Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:31.662684107Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:31.680143563Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:31.680167413Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:31.700052599Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:31.700075849Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:31.717654068Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:31.717678568Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:31.738083122Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:31.738133653Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:31.738363047Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:31.738490879Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:31.769297994Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:31.769325614Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:31.769331044Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:31.769336284Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:31.769342395Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:31.769428646Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"25fcd603-db02-4756-85e7-c0679d60336a","remote_addr":"[fd00:1122:3344:101::1]:36351","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a\", TimeNotSynchronized), (\"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14\", TimeNotSynchronized), (\"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6\", TimeNotSynchronized), (\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\", TimeNotSynchronized), (\"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634\", TimeNotSynchronized)]","latency_us":183284,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:31.769584599Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"6.727424064s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"25fcd603-db02-4756-85e7-c0679d60336a\", \"content-length\": \"124\", \"date\": \"Tue, 27 Aug 2024 19:46:31 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"25fcd603-db02-4756-85e7-c0679d60336a\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:36.990611183Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"329.911898ms"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21e40459-2dec-4da5-8537-0450d95100c7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 27200e24-efc4-4a6c-b7d3-7afde4232881, kind: Service { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a }, name: Name(\"ntp-3f5202b7-215d-4f5c-8883-5d9b359fe25a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 2])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: edf311b1-5e51-4039-b1b2-78828b2fe27a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f7611b-d552-42d2-b5e6-6a3218324db7, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } } } }, OmicronZoneConfig { id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:38.480137902Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:38.480522768Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:38.480534598Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:38.500918202Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:38.500966653Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:38.521399078Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:38.521429578Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:38.539101638Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:38.539131558Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:38.557792324Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:38.557823135Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:38.575930781Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:38.575961772Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:38.59351715Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:38.593544929Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:38.610988766Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:38.611022387Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:38.628744436Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:38.628780046Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:38.62898484Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:38.629117602Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:38.661432692Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:38.661463622Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:38.661470232Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:38.661477772Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:38.661483873Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:38.661564314Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"81493286-75b7-41b9-a548-efc49415c6ec","remote_addr":"[fd00:1122:3344:101::1]:36351","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a\", TimeNotSynchronized), (\"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6\", TimeNotSynchronized), (\"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14\", TimeNotSynchronized), (\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\", TimeNotSynchronized), (\"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634\", TimeNotSynchronized)]","latency_us":181763,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:38.661756157Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"2.892992233s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"81493286-75b7-41b9-a548-efc49415c6ec\", \"content-length\": \"124\", \"date\": \"Tue, 27 Aug 2024 19:46:38 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"81493286-75b7-41b9-a548-efc49415c6ec\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21e40459-2dec-4da5-8537-0450d95100c7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 27200e24-efc4-4a6c-b7d3-7afde4232881, kind: Service { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a }, name: Name(\"ntp-3f5202b7-215d-4f5c-8883-5d9b359fe25a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 2])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: edf311b1-5e51-4039-b1b2-78828b2fe27a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f7611b-d552-42d2-b5e6-6a3218324db7, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } } } }, OmicronZoneConfig { id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:41.547960882Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:41.548335948Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:41.548350748Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:41.569478028Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:41.569501295Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:41.588070339Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:41.588089059Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:41.606598113Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:41.606622203Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:41.62483518Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:41.624858441Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:41.642583921Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:41.642600882Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:41.660383373Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:41.660399223Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:41.677992682Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:41.678011842Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:41.69556611Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:41.69560734Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:41.695825064Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:41.695949156Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:41.7273508Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:41.727400191Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:41.727409781Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:41.727416772Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:41.727422832Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:41.727487773Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"8b461be3-9e2b-4f64-a47f-2ce0c9fddcae","remote_addr":"[fd00:1122:3344:101::1]:36351","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\", TimeNotSynchronized), (\"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6\", TimeNotSynchronized), (\"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14\", TimeNotSynchronized), (\"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a\", TimeNotSynchronized), (\"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634\", TimeNotSynchronized)]","latency_us":179842,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:41.727616775Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"7.994922458s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8b461be3-9e2b-4f64-a47f-2ce0c9fddcae\", \"content-length\": \"124\", \"date\": \"Tue, 27 Aug 2024 19:46:41 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8b461be3-9e2b-4f64-a47f-2ce0c9fddcae\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21e40459-2dec-4da5-8537-0450d95100c7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 27200e24-efc4-4a6c-b7d3-7afde4232881, kind: Service { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a }, name: Name(\"ntp-3f5202b7-215d-4f5c-8883-5d9b359fe25a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 2])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: edf311b1-5e51-4039-b1b2-78828b2fe27a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f7611b-d552-42d2-b5e6-6a3218324db7, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } } } }, OmicronZoneConfig { id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:49.701888875Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:49.702280171Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:49.702287911Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:49.723135133Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:49.723174883Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:49.740775592Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:49.740811772Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:49.759538179Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:49.7595715Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:49.778025932Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:49.778056723Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:49.795829804Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:49.795876275Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:49.814917277Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:49.814950277Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:49.833251537Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:49.833281038Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:49.851224542Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:49.851260982Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:49.851432005Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:49.851530777Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:49.88406408Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:49.884110161Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:49.884117561Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:49.884132861Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:49.884137661Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:49.884179542Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"598a43a1-4aa1-442c-ae44-0617923c5b84","remote_addr":"[fd00:1122:3344:101::1]:36351","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634\", TimeNotSynchronized), (\"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6\", TimeNotSynchronized), (\"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a\", TimeNotSynchronized), (\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\", TimeNotSynchronized), (\"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14\", TimeNotSynchronized)]","latency_us":182620,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:49.884319334Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"8.703776273s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"598a43a1-4aa1-442c-ae44-0617923c5b84\", \"content-length\": \"124\", \"date\": \"Tue, 27 Aug 2024 19:46:49 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"598a43a1-4aa1-442c-ae44-0617923c5b84\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:52.316096953Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"783.222992ms"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:53.714967856Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36317"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:53.715030177Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"b1af7ac2-7de1-44d6-bc22-4a9001cab234","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36317","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":32,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21e40459-2dec-4da5-8537-0450d95100c7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 27200e24-efc4-4a6c-b7d3-7afde4232881, kind: Service { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a }, name: Name(\"ntp-3f5202b7-215d-4f5c-8883-5d9b359fe25a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 2])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: edf311b1-5e51-4039-b1b2-78828b2fe27a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f7611b-d552-42d2-b5e6-6a3218324db7, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } } } }, OmicronZoneConfig { id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:58.565828375Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:58.5661146Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:58.56612968Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:58.587122223Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:58.587151444Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:58.60458533Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:58.60460593Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:58.622649286Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:58.622673556Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:58.642345206Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:58.642364859Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:58.66012911Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:58.66015324Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:58.678371119Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:58.678399799Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:58.696723029Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:58.69675088Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:58.714511161Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:58.714545321Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:58.714751365Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:58.714877437Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:58.747907978Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:58.747951799Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:58.747961259Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:58.747968449Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:58.747975079Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:46:58.748113462Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"01b95326-5d2f-4adc-93ce-632d072163c1","remote_addr":"[fd00:1122:3344:101::1]:36351","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a\", TimeNotSynchronized), (\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\", TimeNotSynchronized), (\"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14\", TimeNotSynchronized), (\"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634\", TimeNotSynchronized), (\"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6\", TimeNotSynchronized)]","latency_us":182639,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:46:58.748307925Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"11.242448794s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"01b95326-5d2f-4adc-93ce-632d072163c1\", \"content-length\": \"124\", \"date\": \"Tue, 27 Aug 2024 19:46:58 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"01b95326-5d2f-4adc-93ce-632d072163c1\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:08.090771371Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"1.659578837s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21e40459-2dec-4da5-8537-0450d95100c7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 27200e24-efc4-4a6c-b7d3-7afde4232881, kind: Service { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a }, name: Name(\"ntp-3f5202b7-215d-4f5c-8883-5d9b359fe25a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 2])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: edf311b1-5e51-4039-b1b2-78828b2fe27a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f7611b-d552-42d2-b5e6-6a3218324db7, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } } } }, OmicronZoneConfig { id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:09.961274595Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:09.96158247Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:09.96159912Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:09.984314612Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:09.984348033Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:10.003330604Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:10.003362645Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:10.022803103Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:10.022840074Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:10.041023522Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:10.041053932Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:10.060539532Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:10.060577082Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:10.078209151Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:10.078240412Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:10.096895517Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:10.096926138Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:10.114277792Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:10.114378554Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:10.11473651Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:10.114872432Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:10.196796285Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:10.196838605Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:10.196846245Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:10.196850475Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:10.196854615Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:10.287690034Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:10.291492426Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:10.295945679Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:10.300396932Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:10.304270566Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:10.308913892Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:10.313389585Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:10.31795675Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:10.322579426Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:10.326981808Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:13.239629725Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:13.239772017Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:13.246475037Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:13.246619149Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:13.306128015Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:13.306248527Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:13.373784083Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:13.373947776Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:13.381697963Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:13.381800365Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a\") failed: Property not found. retry in 62.757057ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:13.394102936Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634\") failed: Property not found. retry in 72.509361ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:13.401907974Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\") failed: Property not found. retry in 69.915766ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:13.453119993Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a\") failed: Property not found. retry in 96.629642ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:13.575255724Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\") failed: Property not found. retry in 67.418773ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:14.07784346Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634\") failed: Property not found. retry in 110.071296ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:14.149103718Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\") failed: Property not found. retry in 169.165218ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:14.198446816Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a\") failed: Property not found. retry in 279.170681ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:14.25483403Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6\") failed: Property not found. retry in 49.250206ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:14.284070429Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634\") failed: Property not found. retry in 288.785629ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:14.316590722Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14\") failed: Property not found. retry in 41.157062ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:14.352885517Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6\") failed: Property not found. retry in 75.751282ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:14.393497042Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\") failed: Property not found. retry in 218.174681ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:14.441808564Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14\") failed: Property not found. retry in 122.114301ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:14.525431735Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6\") failed: Property not found. retry in 293.784909ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:14.607018021Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a\") failed: Property not found. retry in 418.736609ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:14.954319773Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6\") failed: Property not found. retry in 351.764822ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:15.046012275Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634\") failed: Property not found. retry in 312.755812ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:15.170626642Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\") failed: Property not found. retry in 747.880927ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:15.304968109Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14\") failed: Property not found. retry in 106.143959ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:15.42162369Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a\") failed: Property not found. retry in 1.020543619s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:15.458559926Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6\") failed: Property not found. retry in 977.494651ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:15.501691662Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634\") failed: Property not found. retry in 538.904621ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:15.531590272Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14\") failed: Property not found. retry in 520.843039ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:15.607524706Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\") failed: Property not found. retry in 663.492517ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:16.062535123Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634\") failed: Property not found. retry in 754.598826ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:16.078912071Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14\") failed: Property not found. retry in 637.985454ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:16.140615632Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6\") failed: Property not found. retry in 629.743603ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:16.486410529Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a\") failed: Property not found. retry in 1.170033487s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:16.494942398Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\") failed: Property not found. retry in 1.124374469s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:16.734952921Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14\") failed: Property not found. retry in 1.159814021s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:16.786331052Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634\") failed: Property not found. retry in 756.928366ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:16.840594802Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6\") failed: Property not found. retry in 1.193681705s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:17.124796529Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634\") failed: Property not found. retry in 1.438200914s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:17.607903305Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a\") failed: Property not found. retry in 1.411660454s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:17.672459343Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\") failed: Property not found. retry in 1.04032635s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:17.870343479Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14\") failed: Property not found. retry in 1.293885508s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:17.954964402Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6\") failed: Property not found. retry in 1.302575119s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:18.327610929Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\") failed: Property not found. retry in 909.01149ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:18.916305065Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634\") failed: Property not found. retry in 1.211813636s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:19.051173285Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a\") failed: Property not found. retry in 1.061698712s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:19.087940677Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14\") failed: Property not found. retry in 1.032403801s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:19.253096694Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6\") failed: Property not found. retry in 1.308667645s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:19.636149821Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\") failed: Property not found. retry in 726.939303ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:19.832163123Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a\") failed: Property not found. retry in 1.331831902s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:20.155645784Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634\") failed: Property not found. retry in 1.268147118s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:20.270015308Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14\") failed: Property not found. retry in 1.09607044s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:20.292436685Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\") failed: Property not found. retry in 692.066884ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:20.566353903Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6\") failed: Property not found. retry in 702.755333ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:20.951053487Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\") failed: Property not found. retry in 617.688561ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:21.26610764Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14\") failed: Property not found. retry in 1.298345415s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:21.39669684Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634\") failed: Property not found. retry in 872.748075ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:21.547934008Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\") failed: Property not found. retry in 811.576942ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:21.89467681Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\") failed: Property not found. retry in 837.965965ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:22.717259799Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\") failed: Property not found. retry in 810.380442ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:23.6606693Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:23.695610731Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55208"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:23.696594722Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"2529e51c-0b7a-457c-ac85-51380b0b7295","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55208","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":2069,"response_code":"200"} {"msg":"wait for service Some(\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\") failed: Property not found. retry in 778.225828ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:24.762774758Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:24.768604404Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"4.351885661s"} {"msg":"wait for service Some(\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\") failed: Property not found. retry in 563.826598ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:25.552705682Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\") failed: Property not found. retry in 1.365106123s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:26.128511218Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\") failed: Property not found. retry in 1.020158806s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:27.500681573Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\") failed: Property not found. retry in 570.475213ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:28.530148631Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\") failed: Property not found. retry in 639.239386ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:29.110567469Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\") failed: Property not found. retry in 883.552259ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:29.76035446Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7\") failed: Property not found. retry in 1.085174054s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:30.652553743Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:31.759958735Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_cockroachdb_48f7611b-d552-42d2-b5e6-6a3218324db7"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:31.760028586Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_cockroachdb_d7bfd328-d7f8-4feb-a5c5-b91a0977cd14"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:31.760035766Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_cockroachdb_edf311b1-5e51-4039-b1b2-78828b2fe27a"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:31.760041966Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_cockroachdb_0b2cc6c6-593d-4de6-9f46-1ef093a51fc6"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:31.760047836Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_cockroachdb_8dd93bd9-b518-4188-aa7e-c90eeedee634"} {"msg":"Writing ledger to /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:31.760057806Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:31.760622306Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:31.761422079Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"1c43e70d-f311-4e4b-9d8b-9dccebbaf39a","remote_addr":"[fd00:1122:3344:101::1]:36351","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":21849643,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:31.796784807Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:60247"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::4]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:31.825565807Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3681"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:32.000228523Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3699"} {"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:40.777855391Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3722"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:40.777962673Z","hostname":"buskin","pid":1562,"uri":"/cockroachdb","method":"POST","req_id":"500d5c2a-caa4-45c7-a899-b7aef55b5f84","remote_addr":"[fd00:1122:3344:101::1]:60247","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":8981805,"response_code":"204"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 21e40459-2dec-4da5-8537-0450d95100c7, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 27200e24-efc4-4a6c-b7d3-7afde4232881, kind: Service { id: 3f5202b7-215d-4f5c-8883-5d9b359fe25a }, name: Name(\"ntp-3f5202b7-215d-4f5c-8883-5d9b359fe25a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 2])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: edf311b1-5e51-4039-b1b2-78828b2fe27a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f7611b-d552-42d2-b5e6-6a3218324db7, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } } } }, OmicronZoneConfig { id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External } } } }, OmicronZoneConfig { id: 6e6e5549-d8aa-4b92-93c2-2b15052398a6, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 19de1a15-87dd-4421-b9c4-5a26a2f0078f, kind: Service { id: 6e6e5549-d8aa-4b92-93c2-2b15052398a6 }, name: Name(\"external-dns-6e6e5549-d8aa-4b92-93c2-2b15052398a6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 104])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 2591e26a-6588-4335-828d-4938ef1d6397, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 766b280f-b6d1-41b3-9cfc-b05604a1f00a, kind: Service { id: 2591e26a-6588-4335-828d-4938ef1d6397 }, name: Name(\"external-dns-2591e26a-6588-4335-828d-4938ef1d6397\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 172, 0])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: d0fa460e-c334-4a06-8890-451f7039247d, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: a4fc1828-cf68-4cee-9eba-aa2c2011d2a9, kind: Service { id: d0fa460e-c334-4a06-8890-451f7039247d }, name: Name(\"nexus-d0fa460e-c334-4a06-8890-451f7039247d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 221, 63])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: a1b57c17-9dfb-443f-b708-833871354cad, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 1970a6ab-3c09-474a-bf05-c1dbea308c2c, kind: Service { id: a1b57c17-9dfb-443f-b708-833871354cad }, name: Name(\"nexus-a1b57c17-9dfb-443f-b708-833871354cad\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 182, 112])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: fe88893d-1075-499f-81ad-f02ad4112b77, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: e137c875-7fd4-407a-ba2a-2058e5219b83, kind: Service { id: fe88893d-1075-499f-81ad-f02ad4112b77 }, name: Name(\"nexus-fe88893d-1075-499f-81ad-f02ad4112b77\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 203, 34])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2aadd0fc-5177-493a-9f7e-0593c0fe3187, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 060ae63c-2ab8-4f4a-9465-ac7729ee0f85 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 56656cb2-0db5-41a2-a56b-c93b5c15e52d, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } } } }, OmicronZoneConfig { id: 36f087ee-bf6f-4c97-a999-ed642ab3cdab, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 0656eb0b-5e08-42ef-aa7b-308375ba2f5f, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 2b47746b-6454-483c-9ad4-332bf31d0843, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 462cfa74-565a-4b07-9e7f-532c19a0cdef (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0acbec23-eeb8-49c8-b6f7-da12c2e314e7, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External } } } }, OmicronZoneConfig { id: f3a36f2d-8e26-42e8-92e1-a1d2b64e0b1c, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External } } } }, OmicronZoneConfig { id: cbb67d9f-d1a0-4a3c-8318-b095ffabde45, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External } } } }, OmicronZoneConfig { id: 07d87730-3349-4204-8eec-fe9fdab9a6ba, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External } } } }, OmicronZoneConfig { id: f05d7cc2-271e-4b70-9a2d-d625b0ea9fc7, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External } } } }, OmicronZoneConfig { id: 9ba6caf8-787f-4778-a0d1-8b2b954bfcf6, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: f00f974c-b392-4668-a102-c63201ffd653 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f00f974c-b392-4668-a102-c63201ffd653 (zpool), kind: External } } } }, OmicronZoneConfig { id: 79291adc-d002-4588-978b-5e78b8c0ba3c, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 462cfa74-565a-4b07-9e7f-532c19a0cdef (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 462cfa74-565a-4b07-9e7f-532c19a0cdef (zpool), kind: External } } } }, OmicronZoneConfig { id: 8f0afcee-5e8e-470c-86a8-cb826d58e572, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 060ae63c-2ab8-4f4a-9465-ac7729ee0f85 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 060ae63c-2ab8-4f4a-9465-ac7729ee0f85 (zpool), kind: External } } } }, OmicronZoneConfig { id: 91cd64e6-ebb3-41e8-b415-6061da85c422, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 3e23c6bd-257b-4c1d-bcac-865f7970574e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e23c6bd-257b-4c1d-bcac-865f7970574e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:40.813515044Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:40.813648536Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:55030"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:40.814599631Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:40.814606312Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: c9fa5d8b-74a4-4a22-951f-5a84f02a69e3, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:40.837772Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:40.837815841Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: ced4c03d-9859-49c7-a5b2-d6706c87f34e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:40.857001324Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:40.857046505Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 21e40459-2dec-4da5-8537-0450d95100c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:40.877029752Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:40.877067043Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: edf311b1-5e51-4039-b1b2-78828b2fe27a, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:40.894948035Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:40.894982635Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 0b2cc6c6-593d-4de6-9f46-1ef093a51fc6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:40.914762516Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:40.914792948Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 48f7611b-d552-42d2-b5e6-6a3218324db7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:40.93318134Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:40.93322465Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: d7bfd328-d7f8-4feb-a5c5-b91a0977cd14, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:40.951362717Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:40.951403967Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 8dd93bd9-b518-4188-aa7e-c90eeedee634, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e6e5549-d8aa-4b92-93c2-2b15052398a6, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:40.970482039Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:40.97052327Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 6e6e5549-d8aa-4b92-93c2-2b15052398a6, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2591e26a-6588-4335-828d-4938ef1d6397, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:41.059400902Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:41.059457143Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 2591e26a-6588-4335-828d-4938ef1d6397, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56656cb2-0db5-41a2-a56b-c93b5c15e52d, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:41.219039321Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:41.219086132Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 56656cb2-0db5-41a2-a56b-c93b5c15e52d, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0acbec23-eeb8-49c8-b6f7-da12c2e314e7, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:41.3181109Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:41.318160921Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 0acbec23-eeb8-49c8-b6f7-da12c2e314e7, dataset_name: DatasetName { pool_name: ZpoolName { id: d598a515-1659-4717-862f-e7cfd6119c10 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f3a36f2d-8e26-42e8-92e1-a1d2b64e0b1c, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:41.42029126Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:41.420334611Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: f3a36f2d-8e26-42e8-92e1-a1d2b64e0b1c, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b912565-4620-4e7b-8d6f-58182fe5c213 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cbb67d9f-d1a0-4a3c-8318-b095ffabde45, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:41.521581955Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:41.521616815Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: cbb67d9f-d1a0-4a3c-8318-b095ffabde45, dataset_name: DatasetName { pool_name: ZpoolName { id: 7bb5d018-eb30-4a1b-bc97-eb30fe64451c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 07d87730-3349-4204-8eec-fe9fdab9a6ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:41.62652856Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:41.6265843Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 07d87730-3349-4204-8eec-fe9fdab9a6ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 9573bdf9-c5ac-4b32-9886-9d0a0abf58b3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f05d7cc2-271e-4b70-9a2d-d625b0ea9fc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:41.715826439Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:41.71587298Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: f05d7cc2-271e-4b70-9a2d-d625b0ea9fc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 6388c771-f50c-4e79-aa18-35304bc6bf0b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9ba6caf8-787f-4778-a0d1-8b2b954bfcf6, dataset_name: DatasetName { pool_name: ZpoolName { id: f00f974c-b392-4668-a102-c63201ffd653 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:41.811684636Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:41.811733016Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 9ba6caf8-787f-4778-a0d1-8b2b954bfcf6, dataset_name: DatasetName { pool_name: ZpoolName { id: f00f974c-b392-4668-a102-c63201ffd653 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 79291adc-d002-4588-978b-5e78b8c0ba3c, dataset_name: DatasetName { pool_name: ZpoolName { id: 462cfa74-565a-4b07-9e7f-532c19a0cdef (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:41.951373499Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:41.951418009Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 79291adc-d002-4588-978b-5e78b8c0ba3c, dataset_name: DatasetName { pool_name: ZpoolName { id: 462cfa74-565a-4b07-9e7f-532c19a0cdef (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f0afcee-5e8e-470c-86a8-cb826d58e572, dataset_name: DatasetName { pool_name: ZpoolName { id: 060ae63c-2ab8-4f4a-9465-ac7729ee0f85 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.08301479Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.0830624Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 8f0afcee-5e8e-470c-86a8-cb826d58e572, dataset_name: DatasetName { pool_name: ZpoolName { id: 060ae63c-2ab8-4f4a-9465-ac7729ee0f85 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 91cd64e6-ebb3-41e8-b415-6061da85c422, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e23c6bd-257b-4c1d-bcac-865f7970574e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.209638309Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.20968533Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 91cd64e6-ebb3-41e8-b415-6061da85c422, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e23c6bd-257b-4c1d-bcac-865f7970574e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.349483894Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.349539235Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.34979909Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.349984303Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.460492069Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:260"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.460731953Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.460748683Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.460766183Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.474756932Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:260"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.474883314Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.507868423Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.507909254Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.507913194Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.507923254Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.678910538Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:260"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.679151622Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.722538121Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:260"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.722781535Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.722805955Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.722830256Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.722876157Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.724183638Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:260"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.735116237Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Configuring new Omicron zone: oxz_oximeter_2aadd0fc-5177-493a-9f7e-0593c0fe3187","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.735280539Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_36f087ee-bf6f-4c97-a999-ed642ab3cdab","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.740016027Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_f3a36f2d-8e26-42e8-92e1-a1d2b64e0b1c","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.744107914Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_07d87730-3349-4204-8eec-fe9fdab9a6ba","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.749036304Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_79291adc-d002-4588-978b-5e78b8c0ba3c","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.755624642Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_f05d7cc2-271e-4b70-9a2d-d625b0ea9fc7","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.760654264Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_8f0afcee-5e8e-470c-86a8-cb826d58e572","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.766031248Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_2b47746b-6454-483c-9ad4-332bf31d0843","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.770945572Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_oximeter_2aadd0fc-5177-493a-9f7e-0593c0fe3187","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.890876942Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_36f087ee-bf6f-4c97-a999-ed642ab3cdab","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.896582875Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_f3a36f2d-8e26-42e8-92e1-a1d2b64e0b1c","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.901391454Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.939396815Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"} {"msg":"Installing Omicron zone: oxz_crucible_07d87730-3349-4204-8eec-fe9fdab9a6ba","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.939583858Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_79291adc-d002-4588-978b-5e78b8c0ba3c","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.945351743Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_f05d7cc2-271e-4b70-9a2d-d625b0ea9fc7","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.950352794Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_8f0afcee-5e8e-470c-86a8-cb826d58e572","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.955461938Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_2b47746b-6454-483c-9ad4-332bf31d0843","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.960637102Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_clickhouse_56656cb2-0db5-41a2-a56b-c93b5c15e52d","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.966168713Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_0656eb0b-5e08-42ef-aa7b-308375ba2f5f","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.971419609Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_cbb67d9f-d1a0-4a3c-8318-b095ffabde45","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.976787946Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_9ba6caf8-787f-4778-a0d1-8b2b954bfcf6","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.982056762Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_0acbec23-eeb8-49c8-b6f7-da12c2e314e7","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.98678122Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_91cd64e6-ebb3-41e8-b415-6061da85c422","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:42.992259399Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.03454696Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"} {"msg":"Installing Omicron zone: oxz_clickhouse_56656cb2-0db5-41a2-a56b-c93b5c15e52d","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.034755253Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.081324845Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_0656eb0b-5e08-42ef-aa7b-308375ba2f5f","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.081893294Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_cbb67d9f-d1a0-4a3c-8318-b095ffabde45","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.089868534Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_9ba6caf8-787f-4778-a0d1-8b2b954bfcf6","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.097286415Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_0acbec23-eeb8-49c8-b6f7-da12c2e314e7","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.107089866Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_91cd64e6-ebb3-41e8-b415-6061da85c422","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.115764337Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.127123513Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:302"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.127156303Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.177219562Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:424","port":"Port { inner: PortInner(PortData { name: \"opte1\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 221, 63]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.177247292Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1235","zone_type":"nexus"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.244245227Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.24808811Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:302"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.24811838Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.312517933Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:424","port":"Port { inner: PortInner(PortData { name: \"opte2\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 182, 112]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.312545583Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1235","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.314905772Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:302"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.314921912Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.39064796Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:424","port":"Port { inner: PortInner(PortData { name: \"opte3\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 144, 104]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.3906771Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1235","zone_type":"external_dns"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.468508392Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"} {"msg":"no nat entry found for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.47020561Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.47021956Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.473698777Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:302"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.473713597Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.5326961Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:424","port":"Port { inner: PortInner(PortData { name: \"opte4\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 203, 34]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.532734511Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1235","zone_type":"nexus"} {"msg":"creation of nat entry successful for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.533020256Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"no nat entry found for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.534708413Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.534724703Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.534802735Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.534814705Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.535916933Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.536762727Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:302"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.536777167Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.59262258Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:424","port":"Port { inner: PortInner(PortData { name: \"opte5\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 172, 0]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.59265255Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1235","zone_type":"external_dns"} {"msg":"creation of nat entry successful for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.647564228Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"creation of nat entry successful for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.651296009Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.654898578Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.654912698Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"no nat entry found for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.83863311Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.838729772Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"Configuring new Omicron zone: oxz_nexus_d0fa460e-c334-4a06-8890-451f7039247d","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.840489811Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_nexus_a1b57c17-9dfb-443f-b708-833871354cad","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.857685921Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_external_dns_6e6e5549-d8aa-4b92-93c2-2b15052398a6","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.871464887Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"no nat entry found for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.884446209Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.88451149Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.884683683Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.889078685Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Installing Omicron zone: oxz_nexus_d0fa460e-c334-4a06-8890-451f7039247d","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.889241297Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_a1b57c17-9dfb-443f-b708-833871354cad","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.90530203Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_6e6e5549-d8aa-4b92-93c2-2b15052398a6","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.920941185Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"creation of nat entry successful for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.931961575Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:43.991254404Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Configuring new Omicron zone: oxz_nexus_fe88893d-1075-499f-81ad-f02ad4112b77","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:44.046305204Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_external_dns_2591e26a-6588-4335-828d-4938ef1d6397","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:44.068005779Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:47:44.322797202Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"4.070638151s"} {"msg":"Installing Omicron zone: oxz_nexus_fe88893d-1075-499f-81ad-f02ad4112b77","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:44.354423299Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_2591e26a-6588-4335-828d-4938ef1d6397","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:44.509738796Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_crucible_79291adc-d002-4588-978b-5e78b8c0ba3c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:56.05710259Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_8f0afcee-5e8e-470c-86a8-cb826d58e572:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:56.0577403Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_cbb67d9f-d1a0-4a3c-8318-b095ffabde45:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:56.058057946Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_79291adc-d002-4588-978b-5e78b8c0ba3c zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:56.05893974Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_79291adc-d002-4588-978b-5e78b8c0ba3c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_f05d7cc2-271e-4b70-9a2d-d625b0ea9fc7:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:56.81932004Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_8f0afcee-5e8e-470c-86a8-cb826d58e572 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:56.82823641Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_8f0afcee-5e8e-470c-86a8-cb826d58e572","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_cbb67d9f-d1a0-4a3c-8318-b095ffabde45 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:56.92244115Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_cbb67d9f-d1a0-4a3c-8318-b095ffabde45","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_07d87730-3349-4204-8eec-fe9fdab9a6ba:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:57.536666847Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_f05d7cc2-271e-4b70-9a2d-d625b0ea9fc7 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:57.536775729Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_f05d7cc2-271e-4b70-9a2d-d625b0ea9fc7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:58.051283027Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56571"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:58.2815536Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"351afe0b-7ede-437d-9fae-c5ababb5633b","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56571","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":47422,"response_code":"200"} {"msg":"Profile for oxz_crucible_91cd64e6-ebb3-41e8-b415-6061da85c422:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:58.432652979Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_f3a36f2d-8e26-42e8-92e1-a1d2b64e0b1c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:58.482563444Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_07d87730-3349-4204-8eec-fe9fdab9a6ba zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:58.482609025Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_07d87730-3349-4204-8eec-fe9fdab9a6ba","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_pantry_2b47746b-6454-483c-9ad4-332bf31d0843:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:58.654665696Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_pantry_0656eb0b-5e08-42ef-aa7b-308375ba2f5f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:58.655892816Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_9ba6caf8-787f-4778-a0d1-8b2b954bfcf6:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:58.83974251Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_91cd64e6-ebb3-41e8-b415-6061da85c422 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:58.839767191Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_91cd64e6-ebb3-41e8-b415-6061da85c422","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_f3a36f2d-8e26-42e8-92e1-a1d2b64e0b1c zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:58.856017896Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_f3a36f2d-8e26-42e8-92e1-a1d2b64e0b1c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_pantry_2b47746b-6454-483c-9ad4-332bf31d0843 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:58.915180993Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_2b47746b-6454-483c-9ad4-332bf31d0843","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_pantry_0656eb0b-5e08-42ef-aa7b-308375ba2f5f zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:58.924748789Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_0656eb0b-5e08-42ef-aa7b-308375ba2f5f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_9ba6caf8-787f-4778-a0d1-8b2b954bfcf6 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:58.981513197Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9ba6caf8-787f-4778-a0d1-8b2b954bfcf6","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_oximeter_2aadd0fc-5177-493a-9f7e-0593c0fe3187:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:59.958233358Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_oximeter_2aadd0fc-5177-493a-9f7e-0593c0fe3187 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:47:59.974099888Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_2aadd0fc-5177-493a-9f7e-0593c0fe3187","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2b47746b-6454-483c-9ad4-332bf31d0843\") failed: Property not found. retry in 69.291407ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:03.359930657Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_2b47746b-6454-483c-9ad4-332bf31d0843","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f3a36f2d-8e26-42e8-92e1-a1d2b64e0b1c\") failed: Property not found. retry in 34.66181ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:03.371301272Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_f3a36f2d-8e26-42e8-92e1-a1d2b64e0b1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_07d87730-3349-4204-8eec-fe9fdab9a6ba\") failed: Property not found. retry in 53.568406ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:03.380063006Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_07d87730-3349-4204-8eec-fe9fdab9a6ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f05d7cc2-271e-4b70-9a2d-d625b0ea9fc7\") failed: Property not found. retry in 42.126561ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:03.389400448Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_f05d7cc2-271e-4b70-9a2d-d625b0ea9fc7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8f0afcee-5e8e-470c-86a8-cb826d58e572\") failed: Property not found. retry in 55.979289ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:03.398523337Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_8f0afcee-5e8e-470c-86a8-cb826d58e572","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cbb67d9f-d1a0-4a3c-8318-b095ffabde45\") failed: Property not found. retry in 70.102948ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:03.406927665Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_cbb67d9f-d1a0-4a3c-8318-b095ffabde45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_91cd64e6-ebb3-41e8-b415-6061da85c422\") failed: Property not found. retry in 65.61467ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:03.415004626Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_91cd64e6-ebb3-41e8-b415-6061da85c422","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_2aadd0fc-5177-493a-9f7e-0593c0fe3187\") failed: Property not found. retry in 51.507716ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:03.422455858Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_2aadd0fc-5177-493a-9f7e-0593c0fe3187","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f3a36f2d-8e26-42e8-92e1-a1d2b64e0b1c\") failed: Property not found. retry in 119.278943ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:03.429231649Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_f3a36f2d-8e26-42e8-92e1-a1d2b64e0b1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_79291adc-d002-4588-978b-5e78b8c0ba3c\") failed: Property not found. retry in 41.272064ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:03.438689474Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_79291adc-d002-4588-978b-5e78b8c0ba3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0656eb0b-5e08-42ef-aa7b-308375ba2f5f\") failed: Property not found. retry in 61.393875ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:03.449781145Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_0656eb0b-5e08-42ef-aa7b-308375ba2f5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9ba6caf8-787f-4778-a0d1-8b2b954bfcf6\") failed: Property not found. retry in 54.560296ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:03.46051348Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9ba6caf8-787f-4778-a0d1-8b2b954bfcf6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_07d87730-3349-4204-8eec-fe9fdab9a6ba\") failed: Property not found. retry in 85.828541ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:03.471277486Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_07d87730-3349-4204-8eec-fe9fdab9a6ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f05d7cc2-271e-4b70-9a2d-d625b0ea9fc7\") failed: Property not found. retry in 60.192435ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:03.483043278Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_f05d7cc2-271e-4b70-9a2d-d625b0ea9fc7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8f0afcee-5e8e-470c-86a8-cb826d58e572\") failed: Property not found. retry in 94.018487ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:03.493633441Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_8f0afcee-5e8e-470c-86a8-cb826d58e572","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2b47746b-6454-483c-9ad4-332bf31d0843\") failed: Property not found. retry in 109.239947ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:03.502577397Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_2b47746b-6454-483c-9ad4-332bf31d0843","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_2aadd0fc-5177-493a-9f7e-0593c0fe3187\") failed: Property not found. retry in 105.40777ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:03.513230581Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_2aadd0fc-5177-493a-9f7e-0593c0fe3187","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_79291adc-d002-4588-978b-5e78b8c0ba3c\") failed: Property not found. retry in 104.304955ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:03.524723429Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_79291adc-d002-4588-978b-5e78b8c0ba3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0656eb0b-5e08-42ef-aa7b-308375ba2f5f\") failed: Property not found. retry in 133.613975ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:03.533161527Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_0656eb0b-5e08-42ef-aa7b-308375ba2f5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cbb67d9f-d1a0-4a3c-8318-b095ffabde45\") failed: Property not found. retry in 65.684038ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:03.542975737Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_cbb67d9f-d1a0-4a3c-8318-b095ffabde45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9ba6caf8-787f-4778-a0d1-8b2b954bfcf6\") failed: Property not found. retry in 98.614973ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:03.551757471Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9ba6caf8-787f-4778-a0d1-8b2b954bfcf6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_91cd64e6-ebb3-41e8-b415-6061da85c422\") failed: Property not found. retry in 62.044083ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:03.597060271Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_91cd64e6-ebb3-41e8-b415-6061da85c422","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f3a36f2d-8e26-42e8-92e1-a1d2b64e0b1c\") failed: Property not found. retry in 149.176462ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:03.611515187Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_f3a36f2d-8e26-42e8-92e1-a1d2b64e0b1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_07d87730-3349-4204-8eec-fe9fdab9a6ba\") failed: Property not found. retry in 167.298599ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:03.621063593Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_07d87730-3349-4204-8eec-fe9fdab9a6ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f05d7cc2-271e-4b70-9a2d-d625b0ea9fc7\") failed: Property not found. retry in 261.534263ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:03.633414255Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_f05d7cc2-271e-4b70-9a2d-d625b0ea9fc7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8f0afcee-5e8e-470c-86a8-cb826d58e572\") failed: Property not found. retry in 107.316255ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:03.644087839Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_8f0afcee-5e8e-470c-86a8-cb826d58e572","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2b47746b-6454-483c-9ad4-332bf31d0843\") failed: Property not found. retry in 282.379424ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:03.656562683Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_2b47746b-6454-483c-9ad4-332bf31d0843","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cbb67d9f-d1a0-4a3c-8318-b095ffabde45\") failed: Property not found. retry in 297.359943ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:03.669505485Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_cbb67d9f-d1a0-4a3c-8318-b095ffabde45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9ba6caf8-787f-4778-a0d1-8b2b954bfcf6\") failed: Property not found. retry in 288.228231ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:03.682721311Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9ba6caf8-787f-4778-a0d1-8b2b954bfcf6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_91cd64e6-ebb3-41e8-b415-6061da85c422\") failed: Property not found. retry in 208.876782ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:03.692549781Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_91cd64e6-ebb3-41e8-b415-6061da85c422","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_2aadd0fc-5177-493a-9f7e-0593c0fe3187\") failed: Property not found. retry in 171.863426ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:03.705825558Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_2aadd0fc-5177-493a-9f7e-0593c0fe3187","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_79291adc-d002-4588-978b-5e78b8c0ba3c\") failed: Property not found. retry in 247.412607ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:03.718137719Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_79291adc-d002-4588-978b-5e78b8c0ba3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0656eb0b-5e08-42ef-aa7b-308375ba2f5f\") failed: Property not found. retry in 297.790874ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:03.731152962Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_0656eb0b-5e08-42ef-aa7b-308375ba2f5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8f0afcee-5e8e-470c-86a8-cb826d58e572\") failed: Property not found. retry in 278.571365ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:03.764718931Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_8f0afcee-5e8e-470c-86a8-cb826d58e572","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f3a36f2d-8e26-42e8-92e1-a1d2b64e0b1c\") failed: Property not found. retry in 518.997381ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:03.775513597Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_f3a36f2d-8e26-42e8-92e1-a1d2b64e0b1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_07d87730-3349-4204-8eec-fe9fdab9a6ba\") failed: Property not found. retry in 425.287352ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:03.801666902Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_07d87730-3349-4204-8eec-fe9fdab9a6ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_2591e26a-6588-4335-828d-4938ef1d6397:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:03.834247927Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_external_dns_2591e26a-6588-4335-828d-4938ef1d6397 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:03.834650174Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_2591e26a-6588-4335-828d-4938ef1d6397","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_oximeter_2aadd0fc-5177-493a-9f7e-0593c0fe3187\") failed: Property not found. retry in 580.237147ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:03.891182217Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_2aadd0fc-5177-493a-9f7e-0593c0fe3187","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f05d7cc2-271e-4b70-9a2d-d625b0ea9fc7\") failed: Property not found. retry in 321.356017ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:03.906483128Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_f05d7cc2-271e-4b70-9a2d-d625b0ea9fc7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_91cd64e6-ebb3-41e8-b415-6061da85c422\") failed: Property not found. retry in 531.670858ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:03.918447873Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_91cd64e6-ebb3-41e8-b415-6061da85c422","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2b47746b-6454-483c-9ad4-332bf31d0843\") failed: Property not found. retry in 468.874274ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:03.953548866Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_2b47746b-6454-483c-9ad4-332bf31d0843","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_79291adc-d002-4588-978b-5e78b8c0ba3c\") failed: Property not found. retry in 508.722874ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:03.977296185Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_79291adc-d002-4588-978b-5e78b8c0ba3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cbb67d9f-d1a0-4a3c-8318-b095ffabde45\") failed: Property not found. retry in 486.812791ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:03.98799938Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_cbb67d9f-d1a0-4a3c-8318-b095ffabde45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9ba6caf8-787f-4778-a0d1-8b2b954bfcf6\") failed: Property not found. retry in 445.688383ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:03.999350185Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9ba6caf8-787f-4778-a0d1-8b2b954bfcf6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0656eb0b-5e08-42ef-aa7b-308375ba2f5f\") failed: Property not found. retry in 563.507559ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:04.045181254Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_0656eb0b-5e08-42ef-aa7b-308375ba2f5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8f0afcee-5e8e-470c-86a8-cb826d58e572\") failed: Property not found. retry in 565.774873ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:04.058483161Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_8f0afcee-5e8e-470c-86a8-cb826d58e572","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2591e26a-6588-4335-828d-4938ef1d6397\") failed: Property not found. retry in 27.322565ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:04.068638687Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_2591e26a-6588-4335-828d-4938ef1d6397","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2591e26a-6588-4335-828d-4938ef1d6397\") failed: Property not found. retry in 126.760188ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:04.106395395Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_2591e26a-6588-4335-828d-4938ef1d6397","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_07d87730-3349-4204-8eec-fe9fdab9a6ba\") failed: Property not found. retry in 1.076054148s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:04.239582201Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_07d87730-3349-4204-8eec-fe9fdab9a6ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f05d7cc2-271e-4b70-9a2d-d625b0ea9fc7\") failed: Property not found. retry in 549.193721ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:04.252228468Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_f05d7cc2-271e-4b70-9a2d-d625b0ea9fc7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2591e26a-6588-4335-828d-4938ef1d6397\") failed: Property not found. retry in 153.928952ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:04.264384236Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_2591e26a-6588-4335-828d-4938ef1d6397","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f3a36f2d-8e26-42e8-92e1-a1d2b64e0b1c\") failed: Property not found. retry in 1.068430789s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:04.311579897Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_f3a36f2d-8e26-42e8-92e1-a1d2b64e0b1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2591e26a-6588-4335-828d-4938ef1d6397\") failed: Property not found. retry in 514.521691ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:04.431398266Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_2591e26a-6588-4335-828d-4938ef1d6397","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2b47746b-6454-483c-9ad4-332bf31d0843\") failed: Property not found. retry in 765.668897ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:04.445653049Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_2b47746b-6454-483c-9ad4-332bf31d0843","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9ba6caf8-787f-4778-a0d1-8b2b954bfcf6\") failed: Property not found. retry in 1.096844659s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:04.460468211Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9ba6caf8-787f-4778-a0d1-8b2b954bfcf6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_91cd64e6-ebb3-41e8-b415-6061da85c422\") failed: Property not found. retry in 622.848692ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:04.476028205Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_91cd64e6-ebb3-41e8-b415-6061da85c422","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_2aadd0fc-5177-493a-9f7e-0593c0fe3187\") failed: Property not found. retry in 623.547722ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:04.488642001Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_2aadd0fc-5177-493a-9f7e-0593c0fe3187","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cbb67d9f-d1a0-4a3c-8318-b095ffabde45\") failed: Property not found. retry in 924.603759ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:04.501327877Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_cbb67d9f-d1a0-4a3c-8318-b095ffabde45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_79291adc-d002-4588-978b-5e78b8c0ba3c\") failed: Property not found. retry in 1.129009987s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:04.514023431Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_79291adc-d002-4588-978b-5e78b8c0ba3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0656eb0b-5e08-42ef-aa7b-308375ba2f5f\") failed: Property not found. retry in 935.634258ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:04.62325196Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_0656eb0b-5e08-42ef-aa7b-308375ba2f5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8f0afcee-5e8e-470c-86a8-cb826d58e572\") failed: Property not found. retry in 1.257279277s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:04.640036454Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_8f0afcee-5e8e-470c-86a8-cb826d58e572","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f05d7cc2-271e-4b70-9a2d-d625b0ea9fc7\") failed: Property not found. retry in 831.704649ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:04.816809473Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_f05d7cc2-271e-4b70-9a2d-d625b0ea9fc7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2591e26a-6588-4335-828d-4938ef1d6397\") failed: Property not found. retry in 1.02214928s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:04.962640796Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_2591e26a-6588-4335-828d-4938ef1d6397","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_91cd64e6-ebb3-41e8-b415-6061da85c422\") failed: Property not found. retry in 1.174142262s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:05.11036147Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_91cd64e6-ebb3-41e8-b415-6061da85c422","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_2aadd0fc-5177-493a-9f7e-0593c0fe3187\") failed: Property not found. retry in 1.325434126s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:05.12438257Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_2aadd0fc-5177-493a-9f7e-0593c0fe3187","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2b47746b-6454-483c-9ad4-332bf31d0843\") failed: Property not found. retry in 503.397245ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:05.224912513Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_2b47746b-6454-483c-9ad4-332bf31d0843","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_07d87730-3349-4204-8eec-fe9fdab9a6ba\") failed: Property not found. retry in 670.9577ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:05.325568667Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_07d87730-3349-4204-8eec-fe9fdab9a6ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f3a36f2d-8e26-42e8-92e1-a1d2b64e0b1c\") failed: Property not found. retry in 1.41289178s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:05.389934379Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_f3a36f2d-8e26-42e8-92e1-a1d2b64e0b1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cbb67d9f-d1a0-4a3c-8318-b095ffabde45\") failed: Property not found. retry in 888.075829ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:05.436746214Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_cbb67d9f-d1a0-4a3c-8318-b095ffabde45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9ba6caf8-787f-4778-a0d1-8b2b954bfcf6\") failed: Property not found. retry in 679.593493ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:05.567445859Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9ba6caf8-787f-4778-a0d1-8b2b954bfcf6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0656eb0b-5e08-42ef-aa7b-308375ba2f5f\") failed: Property not found. retry in 688.880352ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:05.576792092Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_0656eb0b-5e08-42ef-aa7b-308375ba2f5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_fe88893d-1075-499f-81ad-f02ad4112b77:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:05.597729804Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_nexus_fe88893d-1075-499f-81ad-f02ad4112b77 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:05.597798675Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_fe88893d-1075-499f-81ad-f02ad4112b77","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_79291adc-d002-4588-978b-5e78b8c0ba3c\") failed: Property not found. retry in 853.277675ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:05.653819301Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_79291adc-d002-4588-978b-5e78b8c0ba3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f05d7cc2-271e-4b70-9a2d-d625b0ea9fc7\") failed: Property not found. retry in 1.285224437s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:05.662001944Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_f05d7cc2-271e-4b70-9a2d-d625b0ea9fc7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2b47746b-6454-483c-9ad4-332bf31d0843\") failed: Property not found. retry in 913.100418ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:05.737273225Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_2b47746b-6454-483c-9ad4-332bf31d0843","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fe88893d-1075-499f-81ad-f02ad4112b77\") failed: Property not found. retry in 43.239085ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:05.762272993Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_fe88893d-1075-499f-81ad-f02ad4112b77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fe88893d-1075-499f-81ad-f02ad4112b77\") failed: Property not found. retry in 52.950151ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:05.813313577Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_fe88893d-1075-499f-81ad-f02ad4112b77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fe88893d-1075-499f-81ad-f02ad4112b77\") failed: Property not found. retry in 128.384105ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:05.875360231Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_fe88893d-1075-499f-81ad-f02ad4112b77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8f0afcee-5e8e-470c-86a8-cb826d58e572\") failed: Property not found. retry in 868.732698ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:05.906621882Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_8f0afcee-5e8e-470c-86a8-cb826d58e572","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2591e26a-6588-4335-828d-4938ef1d6397\") failed: Property not found. retry in 1.163974912s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:05.995063867Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_2591e26a-6588-4335-828d-4938ef1d6397","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_07d87730-3349-4204-8eec-fe9fdab9a6ba\") failed: Property not found. retry in 1.360975998s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:06.005489258Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_07d87730-3349-4204-8eec-fe9fdab9a6ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fe88893d-1075-499f-81ad-f02ad4112b77\") failed: Property not found. retry in 505.382129ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:06.014221261Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_fe88893d-1075-499f-81ad-f02ad4112b77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9ba6caf8-787f-4778-a0d1-8b2b954bfcf6\") failed: Property not found. retry in 594.710099ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:06.258039785Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9ba6caf8-787f-4778-a0d1-8b2b954bfcf6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0656eb0b-5e08-42ef-aa7b-308375ba2f5f\") failed: Property not found. retry in 1.383105516s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:06.275263107Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_0656eb0b-5e08-42ef-aa7b-308375ba2f5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_91cd64e6-ebb3-41e8-b415-6061da85c422\") failed: Property not found. retry in 733.147358ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:06.293712268Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_91cd64e6-ebb3-41e8-b415-6061da85c422","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cbb67d9f-d1a0-4a3c-8318-b095ffabde45\") failed: Property not found. retry in 1.245402864s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:06.333441387Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_cbb67d9f-d1a0-4a3c-8318-b095ffabde45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_2aadd0fc-5177-493a-9f7e-0593c0fe3187\") failed: Property not found. retry in 956.440303ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:06.459591269Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_2aadd0fc-5177-493a-9f7e-0593c0fe3187","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_79291adc-d002-4588-978b-5e78b8c0ba3c\") failed: Property not found. retry in 1.344915663s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:06.516573599Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_79291adc-d002-4588-978b-5e78b8c0ba3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fe88893d-1075-499f-81ad-f02ad4112b77\") failed: Property not found. retry in 1.190072169s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:06.52883596Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_fe88893d-1075-499f-81ad-f02ad4112b77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2b47746b-6454-483c-9ad4-332bf31d0843\") failed: Property not found. retry in 1.276220261s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:06.659961452Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_2b47746b-6454-483c-9ad4-332bf31d0843","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8f0afcee-5e8e-470c-86a8-cb826d58e572\") failed: Property not found. retry in 1.197289445s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:06.784728362Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_8f0afcee-5e8e-470c-86a8-cb826d58e572","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f3a36f2d-8e26-42e8-92e1-a1d2b64e0b1c\") failed: Property not found. retry in 519.620114ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:06.81155157Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_f3a36f2d-8e26-42e8-92e1-a1d2b64e0b1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9ba6caf8-787f-4778-a0d1-8b2b954bfcf6\") failed: Property not found. retry in 701.35523ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:06.861504426Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9ba6caf8-787f-4778-a0d1-8b2b954bfcf6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f05d7cc2-271e-4b70-9a2d-d625b0ea9fc7\") failed: Property not found. retry in 1.140135561s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:06.956030831Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_f05d7cc2-271e-4b70-9a2d-d625b0ea9fc7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_91cd64e6-ebb3-41e8-b415-6061da85c422\") failed: Property not found. retry in 795.251ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:07.03614456Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_91cd64e6-ebb3-41e8-b415-6061da85c422","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2591e26a-6588-4335-828d-4938ef1d6397\") failed: Property not found. retry in 677.706849ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:07.261154247Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_2591e26a-6588-4335-828d-4938ef1d6397","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f3a36f2d-8e26-42e8-92e1-a1d2b64e0b1c\") failed: Property not found. retry in 923.674413ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:07.368649324Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_f3a36f2d-8e26-42e8-92e1-a1d2b64e0b1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_07d87730-3349-4204-8eec-fe9fdab9a6ba\") failed: Property not found. retry in 1.219728738s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:07.380735562Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_07d87730-3349-4204-8eec-fe9fdab9a6ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_2aadd0fc-5177-493a-9f7e-0593c0fe3187\") failed: Property not found. retry in 1.33716077s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:07.428798047Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_2aadd0fc-5177-493a-9f7e-0593c0fe3187","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_pantry_36f087ee-bf6f-4c97-a999-ed642ab3cdab:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:07.444370182Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_pantry_36f087ee-bf6f-4c97-a999-ed642ab3cdab zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:07.46934175Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_36f087ee-bf6f-4c97-a999-ed642ab3cdab","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_9ba6caf8-787f-4778-a0d1-8b2b954bfcf6\") failed: Property not found. retry in 562.760846ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:07.574410456Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9ba6caf8-787f-4778-a0d1-8b2b954bfcf6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cbb67d9f-d1a0-4a3c-8318-b095ffabde45\") failed: Property not found. retry in 829.502785ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:07.590922656Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_cbb67d9f-d1a0-4a3c-8318-b095ffabde45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0656eb0b-5e08-42ef-aa7b-308375ba2f5f\") failed: Property not found. retry in 843.511747ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:07.669465239Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_0656eb0b-5e08-42ef-aa7b-308375ba2f5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_36f087ee-bf6f-4c97-a999-ed642ab3cdab\") failed: Property not found. retry in 42.000088ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:07.677942798Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_36f087ee-bf6f-4c97-a999-ed642ab3cdab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fe88893d-1075-499f-81ad-f02ad4112b77\") failed: Property not found. retry in 796.740018ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:07.731157837Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_fe88893d-1075-499f-81ad-f02ad4112b77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_36f087ee-bf6f-4c97-a999-ed642ab3cdab\") failed: Property not found. retry in 119.756859ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:07.741173351Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_36f087ee-bf6f-4c97-a999-ed642ab3cdab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_91cd64e6-ebb3-41e8-b415-6061da85c422\") failed: Property not found. retry in 1.17000904s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:07.844271681Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_91cd64e6-ebb3-41e8-b415-6061da85c422","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_36f087ee-bf6f-4c97-a999-ed642ab3cdab\") failed: Property not found. retry in 155.206937ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:07.872281944Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_36f087ee-bf6f-4c97-a999-ed642ab3cdab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_79291adc-d002-4588-978b-5e78b8c0ba3c\") failed: Property not found. retry in 722.604602ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:07.882536921Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_79291adc-d002-4588-978b-5e78b8c0ba3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2b47746b-6454-483c-9ad4-332bf31d0843\") failed: Property not found. retry in 1.078005105s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:07.949656458Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_2b47746b-6454-483c-9ad4-332bf31d0843","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2591e26a-6588-4335-828d-4938ef1d6397\") failed: Property not found. retry in 1.246188181s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:07.962728582Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_2591e26a-6588-4335-828d-4938ef1d6397","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8f0afcee-5e8e-470c-86a8-cb826d58e572\") failed: Property not found. retry in 962.314147ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:07.996228407Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_8f0afcee-5e8e-470c-86a8-cb826d58e572","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_36f087ee-bf6f-4c97-a999-ed642ab3cdab\") failed: Property not found. retry in 558.175292ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:08.041290646Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_36f087ee-bf6f-4c97-a999-ed642ab3cdab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f05d7cc2-271e-4b70-9a2d-d625b0ea9fc7\") failed: Property not found. retry in 1.386324551s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:08.108109108Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_f05d7cc2-271e-4b70-9a2d-d625b0ea9fc7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9ba6caf8-787f-4778-a0d1-8b2b954bfcf6\") failed: Property not found. retry in 526.797012ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:08.149307284Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9ba6caf8-787f-4778-a0d1-8b2b954bfcf6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f3a36f2d-8e26-42e8-92e1-a1d2b64e0b1c\") failed: Property not found. retry in 1.243181151s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:08.324197019Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_f3a36f2d-8e26-42e8-92e1-a1d2b64e0b1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cbb67d9f-d1a0-4a3c-8318-b095ffabde45\") failed: Property not found. retry in 780.200787ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:08.433837771Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_cbb67d9f-d1a0-4a3c-8318-b095ffabde45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0656eb0b-5e08-42ef-aa7b-308375ba2f5f\") failed: Property not found. retry in 1.015027114s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:08.53052738Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_0656eb0b-5e08-42ef-aa7b-308375ba2f5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fe88893d-1075-499f-81ad-f02ad4112b77\") failed: Property not found. retry in 1.059113225s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:08.544738812Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_fe88893d-1075-499f-81ad-f02ad4112b77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_36f087ee-bf6f-4c97-a999-ed642ab3cdab\") failed: Property not found. retry in 436.703973ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:08.612994233Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_36f087ee-bf6f-4c97-a999-ed642ab3cdab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_07d87730-3349-4204-8eec-fe9fdab9a6ba\") failed: Property not found. retry in 894.987815ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:08.62473835Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_07d87730-3349-4204-8eec-fe9fdab9a6ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_79291adc-d002-4588-978b-5e78b8c0ba3c\") failed: Property not found. retry in 723.464406ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:08.635997884Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_79291adc-d002-4588-978b-5e78b8c0ba3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9ba6caf8-787f-4778-a0d1-8b2b954bfcf6\") failed: Property not found. retry in 1.399835873s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:08.6883883Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9ba6caf8-787f-4778-a0d1-8b2b954bfcf6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_2aadd0fc-5177-493a-9f7e-0593c0fe3187\") failed: Property not found. retry in 1.390040713s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:08.778930179Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_2aadd0fc-5177-493a-9f7e-0593c0fe3187","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8f0afcee-5e8e-470c-86a8-cb826d58e572\") failed: Property not found. retry in 1.117050151s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:08.973489269Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_8f0afcee-5e8e-470c-86a8-cb826d58e572","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_91cd64e6-ebb3-41e8-b415-6061da85c422\") failed: Property not found. retry in 624.88397ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:09.027963329Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_91cd64e6-ebb3-41e8-b415-6061da85c422","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2b47746b-6454-483c-9ad4-332bf31d0843\") failed: Property not found. retry in 925.456096ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:09.042525897Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_2b47746b-6454-483c-9ad4-332bf31d0843","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_36f087ee-bf6f-4c97-a999-ed642ab3cdab\") failed: Property not found. retry in 1.186463756s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:09.064185861Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_36f087ee-bf6f-4c97-a999-ed642ab3cdab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2591e26a-6588-4335-828d-4938ef1d6397\") failed: Property not found. retry in 696.143459ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:09.224818816Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_2591e26a-6588-4335-828d-4938ef1d6397","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cbb67d9f-d1a0-4a3c-8318-b095ffabde45\") failed: Property not found. retry in 626.503619ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:09.237401612Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_cbb67d9f-d1a0-4a3c-8318-b095ffabde45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_79291adc-d002-4588-978b-5e78b8c0ba3c\") failed: Property not found. retry in 990.763472ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:09.372141544Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_79291adc-d002-4588-978b-5e78b8c0ba3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f05d7cc2-271e-4b70-9a2d-d625b0ea9fc7\") failed: Property not found. retry in 895.911883ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:09.507258111Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_f05d7cc2-271e-4b70-9a2d-d625b0ea9fc7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_07d87730-3349-4204-8eec-fe9fdab9a6ba\") failed: Property not found. retry in 821.738002ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:09.532219109Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_07d87730-3349-4204-8eec-fe9fdab9a6ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0656eb0b-5e08-42ef-aa7b-308375ba2f5f\") failed: Property not found. retry in 690.018108ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:09.556206861Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_0656eb0b-5e08-42ef-aa7b-308375ba2f5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f3a36f2d-8e26-42e8-92e1-a1d2b64e0b1c\") failed: Property not found. retry in 1.21677851s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:09.577239535Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_f3a36f2d-8e26-42e8-92e1-a1d2b64e0b1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fe88893d-1075-499f-81ad-f02ad4112b77\") failed: Property not found. retry in 1.232453425s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:09.614819949Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_fe88893d-1075-499f-81ad-f02ad4112b77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_91cd64e6-ebb3-41e8-b415-6061da85c422\") failed: Property not found. retry in 1.270333543s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:09.662700241Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_91cd64e6-ebb3-41e8-b415-6061da85c422","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cbb67d9f-d1a0-4a3c-8318-b095ffabde45\") failed: Property not found. retry in 1.187953613s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:09.872889536Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_cbb67d9f-d1a0-4a3c-8318-b095ffabde45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2591e26a-6588-4335-828d-4938ef1d6397\") failed: Property not found. retry in 532.467029ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:09.930362266Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_2591e26a-6588-4335-828d-4938ef1d6397","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2b47746b-6454-483c-9ad4-332bf31d0843\") failed: Property not found. retry in 828.286349ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:09.978321989Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_2b47746b-6454-483c-9ad4-332bf31d0843","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9ba6caf8-787f-4778-a0d1-8b2b954bfcf6\") failed: Property not found. retry in 946.311889ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:10.09771163Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9ba6caf8-787f-4778-a0d1-8b2b954bfcf6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8f0afcee-5e8e-470c-86a8-cb826d58e572\") failed: Property not found. retry in 1.466271735s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:10.105658389Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_8f0afcee-5e8e-470c-86a8-cb826d58e572","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_2aadd0fc-5177-493a-9f7e-0593c0fe3187\") failed: Property not found. retry in 996.963262ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:10.179348985Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_2aadd0fc-5177-493a-9f7e-0593c0fe3187","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0656eb0b-5e08-42ef-aa7b-308375ba2f5f\") failed: Property not found. retry in 1.214166919s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:10.255252795Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_0656eb0b-5e08-42ef-aa7b-308375ba2f5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_36f087ee-bf6f-4c97-a999-ed642ab3cdab\") failed: Property not found. retry in 1.162548176s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:10.263309807Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_36f087ee-bf6f-4c97-a999-ed642ab3cdab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_07d87730-3349-4204-8eec-fe9fdab9a6ba\") failed: Property not found. retry in 909.506847ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:10.366006945Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_07d87730-3349-4204-8eec-fe9fdab9a6ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_79291adc-d002-4588-978b-5e78b8c0ba3c\") failed: Property not found. retry in 713.283921ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:10.375762365Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_79291adc-d002-4588-978b-5e78b8c0ba3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f05d7cc2-271e-4b70-9a2d-d625b0ea9fc7\") failed: Property not found. retry in 1.477829538s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:10.417027879Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_f05d7cc2-271e-4b70-9a2d-d625b0ea9fc7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2591e26a-6588-4335-828d-4938ef1d6397\") failed: Property not found. retry in 807.356071ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:10.473710725Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_2591e26a-6588-4335-828d-4938ef1d6397","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_6e6e5549-d8aa-4b92-93c2-2b15052398a6:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:10.586532498Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_external_dns_6e6e5549-d8aa-4b92-93c2-2b15052398a6 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:10.586704781Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_6e6e5549-d8aa-4b92-93c2-2b15052398a6","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_0acbec23-eeb8-49c8-b6f7-da12c2e314e7:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:10.714703812Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_0acbec23-eeb8-49c8-b6f7-da12c2e314e7 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:10.715100909Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_0acbec23-eeb8-49c8-b6f7-da12c2e314e7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_external_dns_6e6e5549-d8aa-4b92-93c2-2b15052398a6\") failed: Property not found. retry in 61.90721ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:10.784442202Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_6e6e5549-d8aa-4b92-93c2-2b15052398a6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f3a36f2d-8e26-42e8-92e1-a1d2b64e0b1c\") failed: Property not found. retry in 1.281638638s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:10.807873315Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_f3a36f2d-8e26-42e8-92e1-a1d2b64e0b1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2b47746b-6454-483c-9ad4-332bf31d0843\") failed: Property not found. retry in 767.275868ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:10.827093733Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_2b47746b-6454-483c-9ad4-332bf31d0843","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_6e6e5549-d8aa-4b92-93c2-2b15052398a6\") failed: Property not found. retry in 97.138076ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:10.86570041Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_6e6e5549-d8aa-4b92-93c2-2b15052398a6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fe88893d-1075-499f-81ad-f02ad4112b77\") failed: Property not found. retry in 1.287808064s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:10.920291672Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_fe88893d-1075-499f-81ad-f02ad4112b77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_91cd64e6-ebb3-41e8-b415-6061da85c422\") failed: Property not found. retry in 1.406476518s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:10.958321944Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_91cd64e6-ebb3-41e8-b415-6061da85c422","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_6e6e5549-d8aa-4b92-93c2-2b15052398a6\") failed: Property not found. retry in 167.513758ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:10.973304529Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_6e6e5549-d8aa-4b92-93c2-2b15052398a6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0acbec23-eeb8-49c8-b6f7-da12c2e314e7\") failed: Property not found. retry in 64.402242ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:11.020328887Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_0acbec23-eeb8-49c8-b6f7-da12c2e314e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9ba6caf8-787f-4778-a0d1-8b2b954bfcf6\") failed: Property not found. retry in 651.537027ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:11.062484196Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9ba6caf8-787f-4778-a0d1-8b2b954bfcf6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:11.080502961Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"21.193692362s"} {"msg":"wait for service Some(\"oxz_crucible_cbb67d9f-d1a0-4a3c-8318-b095ffabde45\") failed: Property not found. retry in 1.393125459s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:11.083865071Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_cbb67d9f-d1a0-4a3c-8318-b095ffabde45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0acbec23-eeb8-49c8-b6f7-da12c2e314e7\") failed: Property not found. retry in 51.112695ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:11.097808733Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_0acbec23-eeb8-49c8-b6f7-da12c2e314e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_79291adc-d002-4588-978b-5e78b8c0ba3c\") failed: Property not found. retry in 1.377836113s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:11.108283775Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_79291adc-d002-4588-978b-5e78b8c0ba3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_6e6e5549-d8aa-4b92-93c2-2b15052398a6\") failed: Property not found. retry in 226.775696ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:11.15328046Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_6e6e5549-d8aa-4b92-93c2-2b15052398a6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0acbec23-eeb8-49c8-b6f7-da12c2e314e7\") failed: Property not found. retry in 169.013135ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:11.162268277Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_0acbec23-eeb8-49c8-b6f7-da12c2e314e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_2aadd0fc-5177-493a-9f7e-0593c0fe3187\") failed: Property not found. retry in 928.138605ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:11.187177918Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_2aadd0fc-5177-493a-9f7e-0593c0fe3187","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_07d87730-3349-4204-8eec-fe9fdab9a6ba\") failed: Property not found. retry in 789.672706ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:11.286619159Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_07d87730-3349-4204-8eec-fe9fdab9a6ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2591e26a-6588-4335-828d-4938ef1d6397\") failed: Property not found. retry in 1.437801537s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:11.294945925Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_2591e26a-6588-4335-828d-4938ef1d6397","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0acbec23-eeb8-49c8-b6f7-da12c2e314e7\") failed: Property not found. retry in 583.618321ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:11.342701435Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_0acbec23-eeb8-49c8-b6f7-da12c2e314e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_6e6e5549-d8aa-4b92-93c2-2b15052398a6\") failed: Property not found. retry in 1.165781857s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:11.39070969Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_6e6e5549-d8aa-4b92-93c2-2b15052398a6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_36f087ee-bf6f-4c97-a999-ed642ab3cdab\") failed: Property not found. retry in 1.22413389s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:11.435830968Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_36f087ee-bf6f-4c97-a999-ed642ab3cdab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_d0fa460e-c334-4a06-8890-451f7039247d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:11.452767754Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_nexus_d0fa460e-c334-4a06-8890-451f7039247d zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:11.452835125Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_d0fa460e-c334-4a06-8890-451f7039247d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_nexus_a1b57c17-9dfb-443f-b708-833871354cad:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:11.463726753Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_nexus_a1b57c17-9dfb-443f-b708-833871354cad zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:11.463797985Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_a1b57c17-9dfb-443f-b708-833871354cad","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0656eb0b-5e08-42ef-aa7b-308375ba2f5f\") failed: Property not found. retry in 1.164982099s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:11.48004666Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_0656eb0b-5e08-42ef-aa7b-308375ba2f5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8f0afcee-5e8e-470c-86a8-cb826d58e572\") failed: Property not found. retry in 628.558276ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:11.583820016Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_8f0afcee-5e8e-470c-86a8-cb826d58e572","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2b47746b-6454-483c-9ad4-332bf31d0843\") failed: Property not found. retry in 1.00435005s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:11.607386291Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_2b47746b-6454-483c-9ad4-332bf31d0843","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d0fa460e-c334-4a06-8890-451f7039247d\") failed: Property not found. retry in 41.755448ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:11.625575578Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_d0fa460e-c334-4a06-8890-451f7039247d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_a1b57c17-9dfb-443f-b708-833871354cad\") failed: Property not found. retry in 70.687714ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:11.633065551Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_a1b57c17-9dfb-443f-b708-833871354cad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d0fa460e-c334-4a06-8890-451f7039247d\") failed: Property not found. retry in 148.735548ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:11.675115328Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_d0fa460e-c334-4a06-8890-451f7039247d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_a1b57c17-9dfb-443f-b708-833871354cad\") failed: Property not found. retry in 110.888866ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:11.712897795Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_a1b57c17-9dfb-443f-b708-833871354cad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9ba6caf8-787f-4778-a0d1-8b2b954bfcf6\") failed: Property not found. retry in 1.213703833s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:11.722093726Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9ba6caf8-787f-4778-a0d1-8b2b954bfcf6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d0fa460e-c334-4a06-8890-451f7039247d\") failed: Property not found. retry in 277.775125ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:11.837833407Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_d0fa460e-c334-4a06-8890-451f7039247d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_a1b57c17-9dfb-443f-b708-833871354cad\") failed: Property not found. retry in 135.416871ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:11.847613827Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_a1b57c17-9dfb-443f-b708-833871354cad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f05d7cc2-271e-4b70-9a2d-d625b0ea9fc7\") failed: Property not found. retry in 1.095131199s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:11.91938756Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_f05d7cc2-271e-4b70-9a2d-d625b0ea9fc7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0acbec23-eeb8-49c8-b6f7-da12c2e314e7\") failed: Property not found. retry in 489.022881ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:11.947858625Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_0acbec23-eeb8-49c8-b6f7-da12c2e314e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_a1b57c17-9dfb-443f-b708-833871354cad\") failed: Property not found. retry in 241.661415ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:12.004688454Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_a1b57c17-9dfb-443f-b708-833871354cad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_07d87730-3349-4204-8eec-fe9fdab9a6ba\") failed: Property not found. retry in 1.439397591s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:12.087029539Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_07d87730-3349-4204-8eec-fe9fdab9a6ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f3a36f2d-8e26-42e8-92e1-a1d2b64e0b1c\") failed: Property not found. retry in 1.179314859s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:12.098567608Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_f3a36f2d-8e26-42e8-92e1-a1d2b64e0b1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_2aadd0fc-5177-493a-9f7e-0593c0fe3187\") failed: Property not found. retry in 518.365871ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:12.126368972Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_2aadd0fc-5177-493a-9f7e-0593c0fe3187","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d0fa460e-c334-4a06-8890-451f7039247d\") failed: Property not found. retry in 415.026025ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:12.134501135Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_d0fa460e-c334-4a06-8890-451f7039247d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fe88893d-1075-499f-81ad-f02ad4112b77\") failed: Property not found. retry in 1.180847804s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:12.218924025Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_fe88893d-1075-499f-81ad-f02ad4112b77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8f0afcee-5e8e-470c-86a8-cb826d58e572\") failed: Property not found. retry in 936.188458ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:12.227557006Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_8f0afcee-5e8e-470c-86a8-cb826d58e572","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_a1b57c17-9dfb-443f-b708-833871354cad\") failed: Property not found. retry in 686.884828ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:12.255501133Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_a1b57c17-9dfb-443f-b708-833871354cad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_91cd64e6-ebb3-41e8-b415-6061da85c422\") failed: Property not found. retry in 639.642724ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:12.374998336Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_91cd64e6-ebb3-41e8-b415-6061da85c422","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0acbec23-eeb8-49c8-b6f7-da12c2e314e7\") failed: Property not found. retry in 837.813956ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:12.44760306Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_0acbec23-eeb8-49c8-b6f7-da12c2e314e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cbb67d9f-d1a0-4a3c-8318-b095ffabde45\") failed: Property not found. retry in 1.184797746s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:12.496454769Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_cbb67d9f-d1a0-4a3c-8318-b095ffabde45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_79291adc-d002-4588-978b-5e78b8c0ba3c\") failed: Property not found. retry in 1.373352051s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:12.506622446Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_79291adc-d002-4588-978b-5e78b8c0ba3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d0fa460e-c334-4a06-8890-451f7039247d\") failed: Property not found. retry in 751.021316ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:12.560688219Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_d0fa460e-c334-4a06-8890-451f7039247d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_6e6e5549-d8aa-4b92-93c2-2b15052398a6\") failed: Property not found. retry in 1.369407803s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:12.569233889Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_6e6e5549-d8aa-4b92-93c2-2b15052398a6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2b47746b-6454-483c-9ad4-332bf31d0843\") failed: Property not found. retry in 736.343231ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:12.620899323Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_2b47746b-6454-483c-9ad4-332bf31d0843","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_2aadd0fc-5177-493a-9f7e-0593c0fe3187\") failed: Property not found. retry in 1.320890923s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:12.653044588Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_2aadd0fc-5177-493a-9f7e-0593c0fe3187","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0656eb0b-5e08-42ef-aa7b-308375ba2f5f\") failed: Property not found. retry in 610.731065ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:12.660585802Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_0656eb0b-5e08-42ef-aa7b-308375ba2f5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_36f087ee-bf6f-4c97-a999-ed642ab3cdab\") failed: Property not found. retry in 1.249334581s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:12.668807186Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_36f087ee-bf6f-4c97-a999-ed642ab3cdab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2591e26a-6588-4335-828d-4938ef1d6397\") failed: Property not found. retry in 549.758226ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:12.744791048Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_2591e26a-6588-4335-828d-4938ef1d6397","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9ba6caf8-787f-4778-a0d1-8b2b954bfcf6\") failed: Property not found. retry in 1.468938902s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:12.949542364Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9ba6caf8-787f-4778-a0d1-8b2b954bfcf6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_a1b57c17-9dfb-443f-b708-833871354cad\") failed: Property not found. retry in 817.330661ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:12.961254485Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_a1b57c17-9dfb-443f-b708-833871354cad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f05d7cc2-271e-4b70-9a2d-d625b0ea9fc7\") failed: Property not found. retry in 1.095534364s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:13.02886926Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_f05d7cc2-271e-4b70-9a2d-d625b0ea9fc7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_91cd64e6-ebb3-41e8-b415-6061da85c422\") failed: Property not found. retry in 791.068706ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:13.040865686Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_91cd64e6-ebb3-41e8-b415-6061da85c422","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8f0afcee-5e8e-470c-86a8-cb826d58e572\") failed: Property not found. retry in 1.498321669s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:13.176766397Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_8f0afcee-5e8e-470c-86a8-cb826d58e572","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0656eb0b-5e08-42ef-aa7b-308375ba2f5f\") failed: Property not found. retry in 523.059098ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:13.286214356Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_0656eb0b-5e08-42ef-aa7b-308375ba2f5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f3a36f2d-8e26-42e8-92e1-a1d2b64e0b1c\") failed: Property not found. retry in 1.475474028s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:13.299632795Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_f3a36f2d-8e26-42e8-92e1-a1d2b64e0b1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0acbec23-eeb8-49c8-b6f7-da12c2e314e7\") failed: Property not found. retry in 1.14487011s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:13.312200751Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_0acbec23-eeb8-49c8-b6f7-da12c2e314e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d0fa460e-c334-4a06-8890-451f7039247d\") failed: Property not found. retry in 1.26797125s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:13.327967968Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_d0fa460e-c334-4a06-8890-451f7039247d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2591e26a-6588-4335-828d-4938ef1d6397\") failed: Property not found. retry in 1.314283724s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:13.343668055Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_2591e26a-6588-4335-828d-4938ef1d6397","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2b47746b-6454-483c-9ad4-332bf31d0843\") failed: Property not found. retry in 1.026630719s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:13.370455093Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_2b47746b-6454-483c-9ad4-332bf31d0843","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fe88893d-1075-499f-81ad-f02ad4112b77\") failed: Property not found. retry in 1.438787922s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:13.414255818Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_fe88893d-1075-499f-81ad-f02ad4112b77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_07d87730-3349-4204-8eec-fe9fdab9a6ba\") failed: Property not found. retry in 701.745333ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:13.545062325Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_07d87730-3349-4204-8eec-fe9fdab9a6ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cbb67d9f-d1a0-4a3c-8318-b095ffabde45\") failed: Property not found. retry in 1.159380329s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:13.697240032Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_cbb67d9f-d1a0-4a3c-8318-b095ffabde45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_a1b57c17-9dfb-443f-b708-833871354cad\") failed: Property not found. retry in 723.921809ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:13.789822915Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_a1b57c17-9dfb-443f-b708-833871354cad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0656eb0b-5e08-42ef-aa7b-308375ba2f5f\") failed: Property not found. retry in 1.07008817s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:13.822895216Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_0656eb0b-5e08-42ef-aa7b-308375ba2f5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_91cd64e6-ebb3-41e8-b415-6061da85c422\") failed: Property not found. retry in 1.149119503s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:13.844812824Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_91cd64e6-ebb3-41e8-b415-6061da85c422","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_79291adc-d002-4588-978b-5e78b8c0ba3c\") failed: Property not found. retry in 1.139656309s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:13.892349201Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_79291adc-d002-4588-978b-5e78b8c0ba3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_36f087ee-bf6f-4c97-a999-ed642ab3cdab\") failed: Property not found. retry in 1.457318639s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:13.932734731Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_36f087ee-bf6f-4c97-a999-ed642ab3cdab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_6e6e5549-d8aa-4b92-93c2-2b15052398a6\") failed: Property not found. retry in 589.361975ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:13.953153994Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_6e6e5549-d8aa-4b92-93c2-2b15052398a6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_2aadd0fc-5177-493a-9f7e-0593c0fe3187\") failed: Property not found. retry in 1.340021716s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:13.992816293Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_2aadd0fc-5177-493a-9f7e-0593c0fe3187","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f05d7cc2-271e-4b70-9a2d-d625b0ea9fc7\") failed: Property not found. retry in 951.177216ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:14.140560567Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_f05d7cc2-271e-4b70-9a2d-d625b0ea9fc7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_07d87730-3349-4204-8eec-fe9fdab9a6ba\") failed: Property not found. retry in 700.72878ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:14.259149605Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_07d87730-3349-4204-8eec-fe9fdab9a6ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2b47746b-6454-483c-9ad4-332bf31d0843\") failed: Property not found. retry in 694.631305ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:14.415770815Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_2b47746b-6454-483c-9ad4-332bf31d0843","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9ba6caf8-787f-4778-a0d1-8b2b954bfcf6\") failed: Property not found. retry in 1.166011339s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:14.435136991Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9ba6caf8-787f-4778-a0d1-8b2b954bfcf6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0acbec23-eeb8-49c8-b6f7-da12c2e314e7\") failed: Property not found. retry in 1.194743512s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:14.470395097Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_0acbec23-eeb8-49c8-b6f7-da12c2e314e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_clickhouse_56656cb2-0db5-41a2-a56b-c93b5c15e52d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:14.502522362Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_clickhouse_56656cb2-0db5-41a2-a56b-c93b5c15e52d zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:14.502751065Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_56656cb2-0db5-41a2-a56b-c93b5c15e52d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_nexus_a1b57c17-9dfb-443f-b708-833871354cad\") failed: Property not found. retry in 1.124653662s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:14.528260562Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_a1b57c17-9dfb-443f-b708-833871354cad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_6e6e5549-d8aa-4b92-93c2-2b15052398a6\") failed: Property not found. retry in 1.237879918s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:14.556660846Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_6e6e5549-d8aa-4b92-93c2-2b15052398a6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d0fa460e-c334-4a06-8890-451f7039247d\") failed: Property not found. retry in 993.703081ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:14.609991862Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_d0fa460e-c334-4a06-8890-451f7039247d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2591e26a-6588-4335-828d-4938ef1d6397\") failed: Property not found. retry in 867.947944ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:14.671572534Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_2591e26a-6588-4335-828d-4938ef1d6397","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8f0afcee-5e8e-470c-86a8-cb826d58e572\") failed: Property not found. retry in 521.041206ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:14.691246266Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_8f0afcee-5e8e-470c-86a8-cb826d58e572","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_56656cb2-0db5-41a2-a56b-c93b5c15e52d\") failed: Property not found. retry in 52.264802ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:14.757486678Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_56656cb2-0db5-41a2-a56b-c93b5c15e52d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f3a36f2d-8e26-42e8-92e1-a1d2b64e0b1c\") failed: Property not found. retry in 668.011981ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:14.785953934Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_f3a36f2d-8e26-42e8-92e1-a1d2b64e0b1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_56656cb2-0db5-41a2-a56b-c93b5c15e52d\") failed: Property not found. retry in 134.231006ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:14.821906171Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_56656cb2-0db5-41a2-a56b-c93b5c15e52d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fe88893d-1075-499f-81ad-f02ad4112b77\") failed: Property not found. retry in 912.734401ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:14.870911342Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_fe88893d-1075-499f-81ad-f02ad4112b77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cbb67d9f-d1a0-4a3c-8318-b095ffabde45\") failed: Property not found. retry in 1.140434186s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:14.889597267Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_cbb67d9f-d1a0-4a3c-8318-b095ffabde45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0656eb0b-5e08-42ef-aa7b-308375ba2f5f\") failed: Property not found. retry in 607.156846ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:14.909740976Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_0656eb0b-5e08-42ef-aa7b-308375ba2f5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_56656cb2-0db5-41a2-a56b-c93b5c15e52d\") failed: Property not found. retry in 100.949976ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:14.972077705Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_56656cb2-0db5-41a2-a56b-c93b5c15e52d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_07d87730-3349-4204-8eec-fe9fdab9a6ba\") failed: Property not found. retry in 1.121381824s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:14.99012299Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_07d87730-3349-4204-8eec-fe9fdab9a6ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_91cd64e6-ebb3-41e8-b415-6061da85c422\") failed: Property not found. retry in 1.147956448s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:15.01523681Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_91cd64e6-ebb3-41e8-b415-6061da85c422","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_79291adc-d002-4588-978b-5e78b8c0ba3c\") failed: Property not found. retry in 1.062442599s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:15.045480835Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_79291adc-d002-4588-978b-5e78b8c0ba3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_56656cb2-0db5-41a2-a56b-c93b5c15e52d\") failed: Property not found. retry in 213.929282ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:15.090499811Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_56656cb2-0db5-41a2-a56b-c93b5c15e52d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f05d7cc2-271e-4b70-9a2d-d625b0ea9fc7\") failed: Property not found. retry in 1.248769988s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:15.105030448Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_f05d7cc2-271e-4b70-9a2d-d625b0ea9fc7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2b47746b-6454-483c-9ad4-332bf31d0843\") failed: Property not found. retry in 876.298814ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:15.122138697Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_2b47746b-6454-483c-9ad4-332bf31d0843","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8f0afcee-5e8e-470c-86a8-cb826d58e572\") failed: Property not found. retry in 816.337807ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:15.224274677Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_8f0afcee-5e8e-470c-86a8-cb826d58e572","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_56656cb2-0db5-41a2-a56b-c93b5c15e52d\") failed: Property not found. retry in 632.467195ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:15.317085273Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_56656cb2-0db5-41a2-a56b-c93b5c15e52d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_2aadd0fc-5177-493a-9f7e-0593c0fe3187\") failed: Property not found. retry in 665.725741ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:15.341617044Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_2aadd0fc-5177-493a-9f7e-0593c0fe3187","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_36f087ee-bf6f-4c97-a999-ed642ab3cdab\") failed: Property not found. retry in 824.809642ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:15.40070958Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_36f087ee-bf6f-4c97-a999-ed642ab3cdab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f3a36f2d-8e26-42e8-92e1-a1d2b64e0b1c\") failed: Property not found. retry in 1.328514446s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:15.463854832Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_f3a36f2d-8e26-42e8-92e1-a1d2b64e0b1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0656eb0b-5e08-42ef-aa7b-308375ba2f5f\") failed: Property not found. retry in 1.135440351s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:15.529082547Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_0656eb0b-5e08-42ef-aa7b-308375ba2f5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2591e26a-6588-4335-828d-4938ef1d6397\") failed: Property not found. retry in 785.485739ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:15.55068906Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_2591e26a-6588-4335-828d-4938ef1d6397","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9ba6caf8-787f-4778-a0d1-8b2b954bfcf6\") failed: Property not found. retry in 812.399932ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:15.613279243Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9ba6caf8-787f-4778-a0d1-8b2b954bfcf6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d0fa460e-c334-4a06-8890-451f7039247d\") failed: Property not found. retry in 726.514275ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:15.626219994Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_d0fa460e-c334-4a06-8890-451f7039247d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_a1b57c17-9dfb-443f-b708-833871354cad\") failed: Property not found. retry in 1.037429457s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:15.664397008Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_a1b57c17-9dfb-443f-b708-833871354cad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0acbec23-eeb8-49c8-b6f7-da12c2e314e7\") failed: Property not found. retry in 1.403920772s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:15.676814081Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_0acbec23-eeb8-49c8-b6f7-da12c2e314e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fe88893d-1075-499f-81ad-f02ad4112b77\") failed: Property not found. retry in 916.013358ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:15.797738197Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_fe88893d-1075-499f-81ad-f02ad4112b77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_6e6e5549-d8aa-4b92-93c2-2b15052398a6\") failed: Property not found. retry in 893.565441ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:15.811643475Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_6e6e5549-d8aa-4b92-93c2-2b15052398a6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_56656cb2-0db5-41a2-a56b-c93b5c15e52d\") failed: Property not found. retry in 1.217297243s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:15.962929267Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_56656cb2-0db5-41a2-a56b-c93b5c15e52d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2b47746b-6454-483c-9ad4-332bf31d0843\") failed: Property not found. retry in 927.211533ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:16.013219359Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_2b47746b-6454-483c-9ad4-332bf31d0843","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_2aadd0fc-5177-493a-9f7e-0593c0fe3187\") failed: Property not found. retry in 529.033768ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:16.027113746Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_2aadd0fc-5177-493a-9f7e-0593c0fe3187","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cbb67d9f-d1a0-4a3c-8318-b095ffabde45\") failed: Property not found. retry in 681.019959ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:16.04019952Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_cbb67d9f-d1a0-4a3c-8318-b095ffabde45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8f0afcee-5e8e-470c-86a8-cb826d58e572\") failed: Property not found. retry in 568.688868ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:16.051025552Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_8f0afcee-5e8e-470c-86a8-cb826d58e572","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_79291adc-d002-4588-978b-5e78b8c0ba3c\") failed: Property not found. retry in 1.493697782s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:16.12095581Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_79291adc-d002-4588-978b-5e78b8c0ba3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_07d87730-3349-4204-8eec-fe9fdab9a6ba\") failed: Property not found. retry in 970.920202ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:16.135143381Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_07d87730-3349-4204-8eec-fe9fdab9a6ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_91cd64e6-ebb3-41e8-b415-6061da85c422\") failed: Property not found. retry in 1.389113681s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:16.178211605Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_91cd64e6-ebb3-41e8-b415-6061da85c422","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_36f087ee-bf6f-4c97-a999-ed642ab3cdab\") failed: Property not found. retry in 1.437162487s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:16.238354988Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_36f087ee-bf6f-4c97-a999-ed642ab3cdab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2591e26a-6588-4335-828d-4938ef1d6397\") failed: Property not found. retry in 1.05151602s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:16.346917162Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_2591e26a-6588-4335-828d-4938ef1d6397","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d0fa460e-c334-4a06-8890-451f7039247d\") failed: Property not found. retry in 1.268882946s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:16.362825532Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_d0fa460e-c334-4a06-8890-451f7039247d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f05d7cc2-271e-4b70-9a2d-d625b0ea9fc7\") failed: Property not found. retry in 711.518187ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:16.37245008Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_f05d7cc2-271e-4b70-9a2d-d625b0ea9fc7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9ba6caf8-787f-4778-a0d1-8b2b954bfcf6\") failed: Property not found. retry in 1.440233784s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:16.439901182Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9ba6caf8-787f-4778-a0d1-8b2b954bfcf6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_2aadd0fc-5177-493a-9f7e-0593c0fe3187\") failed: Property not found. retry in 810.134311ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:16.568402031Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_2aadd0fc-5177-493a-9f7e-0593c0fe3187","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8f0afcee-5e8e-470c-86a8-cb826d58e572\") failed: Property not found. retry in 642.308197ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:16.631262118Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_8f0afcee-5e8e-470c-86a8-cb826d58e572","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0656eb0b-5e08-42ef-aa7b-308375ba2f5f\") failed: Property not found. retry in 1.132562576s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:16.676526428Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_0656eb0b-5e08-42ef-aa7b-308375ba2f5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_a1b57c17-9dfb-443f-b708-833871354cad\") failed: Property not found. retry in 1.357304789s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:16.717365945Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_a1b57c17-9dfb-443f-b708-833871354cad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_6e6e5549-d8aa-4b92-93c2-2b15052398a6\") failed: Property not found. retry in 826.085293ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:16.72989207Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_6e6e5549-d8aa-4b92-93c2-2b15052398a6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fe88893d-1075-499f-81ad-f02ad4112b77\") failed: Property not found. retry in 593.161768ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:16.741871036Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_fe88893d-1075-499f-81ad-f02ad4112b77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cbb67d9f-d1a0-4a3c-8318-b095ffabde45\") failed: Property not found. retry in 1.029827836s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:16.752451539Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_cbb67d9f-d1a0-4a3c-8318-b095ffabde45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f3a36f2d-8e26-42e8-92e1-a1d2b64e0b1c\") failed: Property not found. retry in 938.983601ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:16.813009558Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_f3a36f2d-8e26-42e8-92e1-a1d2b64e0b1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2b47746b-6454-483c-9ad4-332bf31d0843\") failed: Property not found. retry in 576.319978ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:16.954336908Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_2b47746b-6454-483c-9ad4-332bf31d0843","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0acbec23-eeb8-49c8-b6f7-da12c2e314e7\") failed: Property not found. retry in 583.202851ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:17.09518373Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_0acbec23-eeb8-49c8-b6f7-da12c2e314e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f05d7cc2-271e-4b70-9a2d-d625b0ea9fc7\") failed: Property not found. retry in 1.371235388s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:17.107034733Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_f05d7cc2-271e-4b70-9a2d-d625b0ea9fc7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_07d87730-3349-4204-8eec-fe9fdab9a6ba\") failed: Property not found. retry in 509.555891ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:17.1190352Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_07d87730-3349-4204-8eec-fe9fdab9a6ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_56656cb2-0db5-41a2-a56b-c93b5c15e52d\") failed: Property not found. retry in 1.177281655s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:17.197703905Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_56656cb2-0db5-41a2-a56b-c93b5c15e52d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8f0afcee-5e8e-470c-86a8-cb826d58e572\") failed: Property not found. retry in 697.499753ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:17.28977825Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_8f0afcee-5e8e-470c-86a8-cb826d58e572","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fe88893d-1075-499f-81ad-f02ad4112b77\") failed: Property not found. retry in 690.539476ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:17.348459279Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_fe88893d-1075-499f-81ad-f02ad4112b77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_2aadd0fc-5177-493a-9f7e-0593c0fe3187\") failed: Property not found. retry in 937.621291ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:17.392725912Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_2aadd0fc-5177-493a-9f7e-0593c0fe3187","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2591e26a-6588-4335-828d-4938ef1d6397\") failed: Property not found. retry in 913.397287ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:17.413317409Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_2591e26a-6588-4335-828d-4938ef1d6397","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2b47746b-6454-483c-9ad4-332bf31d0843\") failed: Property not found. retry in 1.031486871s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:17.543855991Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_2b47746b-6454-483c-9ad4-332bf31d0843","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_6e6e5549-d8aa-4b92-93c2-2b15052398a6\") failed: Property not found. retry in 1.207911711s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:17.575383136Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_6e6e5549-d8aa-4b92-93c2-2b15052398a6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_91cd64e6-ebb3-41e8-b415-6061da85c422\") failed: Property not found. retry in 1.142468707s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:17.589899914Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_91cd64e6-ebb3-41e8-b415-6061da85c422","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_79291adc-d002-4588-978b-5e78b8c0ba3c\") failed: Property not found. retry in 743.844128ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:17.633927253Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_79291adc-d002-4588-978b-5e78b8c0ba3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d0fa460e-c334-4a06-8890-451f7039247d\") failed: Property not found. retry in 1.425676745s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:17.646326656Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_d0fa460e-c334-4a06-8890-451f7039247d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_07d87730-3349-4204-8eec-fe9fdab9a6ba\") failed: Property not found. retry in 1.061997764s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:17.658156849Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_07d87730-3349-4204-8eec-fe9fdab9a6ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_36f087ee-bf6f-4c97-a999-ed642ab3cdab\") failed: Property not found. retry in 1.250129018s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:17.687633891Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_36f087ee-bf6f-4c97-a999-ed642ab3cdab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0acbec23-eeb8-49c8-b6f7-da12c2e314e7\") failed: Property not found. retry in 810.98087ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:17.701935885Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_0acbec23-eeb8-49c8-b6f7-da12c2e314e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f3a36f2d-8e26-42e8-92e1-a1d2b64e0b1c\") failed: Property not found. retry in 629.286743ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:17.764652499Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_f3a36f2d-8e26-42e8-92e1-a1d2b64e0b1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cbb67d9f-d1a0-4a3c-8318-b095ffabde45\") failed: Property not found. retry in 1.499467424s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:17.7934158Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_cbb67d9f-d1a0-4a3c-8318-b095ffabde45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0656eb0b-5e08-42ef-aa7b-308375ba2f5f\") failed: Property not found. retry in 939.453157ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:17.820497822Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_0656eb0b-5e08-42ef-aa7b-308375ba2f5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9ba6caf8-787f-4778-a0d1-8b2b954bfcf6\") failed: Property not found. retry in 1.430510351s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:17.89011158Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9ba6caf8-787f-4778-a0d1-8b2b954bfcf6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8f0afcee-5e8e-470c-86a8-cb826d58e572\") failed: Property not found. retry in 1.236443264s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:18.001274916Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_8f0afcee-5e8e-470c-86a8-cb826d58e572","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fe88893d-1075-499f-81ad-f02ad4112b77\") failed: Property not found. retry in 1.199780199s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:18.050249083Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_fe88893d-1075-499f-81ad-f02ad4112b77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_a1b57c17-9dfb-443f-b708-833871354cad\") failed: Property not found. retry in 1.372015928s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:18.087596117Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_a1b57c17-9dfb-443f-b708-833871354cad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2591e26a-6588-4335-828d-4938ef1d6397\") failed: Property not found. retry in 504.684165ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:18.340373448Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_2591e26a-6588-4335-828d-4938ef1d6397","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_2aadd0fc-5177-493a-9f7e-0593c0fe3187\") failed: Property not found. retry in 1.281149383s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:18.349585229Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_2aadd0fc-5177-493a-9f7e-0593c0fe3187","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_56656cb2-0db5-41a2-a56b-c93b5c15e52d\") failed: Property not found. retry in 942.241893ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:18.386574913Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_56656cb2-0db5-41a2-a56b-c93b5c15e52d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f3a36f2d-8e26-42e8-92e1-a1d2b64e0b1c\") failed: Property not found. retry in 1.088790807s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:18.413371121Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_f3a36f2d-8e26-42e8-92e1-a1d2b64e0b1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f05d7cc2-271e-4b70-9a2d-d625b0ea9fc7\") failed: Property not found. retry in 1.437007579s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:18.491899083Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_f05d7cc2-271e-4b70-9a2d-d625b0ea9fc7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0acbec23-eeb8-49c8-b6f7-da12c2e314e7\") failed: Property not found. retry in 523.562399ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:18.524496656Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_0acbec23-eeb8-49c8-b6f7-da12c2e314e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_07d87730-3349-4204-8eec-fe9fdab9a6ba\") failed: Property not found. retry in 675.024558ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:18.731622011Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_07d87730-3349-4204-8eec-fe9fdab9a6ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_91cd64e6-ebb3-41e8-b415-6061da85c422\") failed: Property not found. retry in 780.555971ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:18.7422457Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_91cd64e6-ebb3-41e8-b415-6061da85c422","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_6e6e5549-d8aa-4b92-93c2-2b15052398a6\") failed: Property not found. retry in 852.590127ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:18.797451267Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_6e6e5549-d8aa-4b92-93c2-2b15052398a6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_36f087ee-bf6f-4c97-a999-ed642ab3cdab\") failed: Property not found. retry in 1.159241422s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:18.953852903Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_36f087ee-bf6f-4c97-a999-ed642ab3cdab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0acbec23-eeb8-49c8-b6f7-da12c2e314e7\") failed: Property not found. retry in 1.354757631s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:19.063791829Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_0acbec23-eeb8-49c8-b6f7-da12c2e314e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d0fa460e-c334-4a06-8890-451f7039247d\") failed: Property not found. retry in 658.166332ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:19.087865163Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_d0fa460e-c334-4a06-8890-451f7039247d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_56656cb2-0db5-41a2-a56b-c93b5c15e52d\") failed: Property not found. retry in 822.182723ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:19.363122781Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_56656cb2-0db5-41a2-a56b-c93b5c15e52d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_a1b57c17-9dfb-443f-b708-833871354cad\") failed: Property not found. retry in 1.061397716s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:19.475444417Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_a1b57c17-9dfb-443f-b708-833871354cad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_6e6e5549-d8aa-4b92-93c2-2b15052398a6\") failed: Property not found. retry in 851.932501ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:19.689446774Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_6e6e5549-d8aa-4b92-93c2-2b15052398a6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_56656cb2-0db5-41a2-a56b-c93b5c15e52d\") failed: Property not found. retry in 556.729713ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:20.206323661Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_56656cb2-0db5-41a2-a56b-c93b5c15e52d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_56656cb2-0db5-41a2-a56b-c93b5c15e52d\") failed: Property not found. retry in 856.588711ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:20.833844415Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_56656cb2-0db5-41a2-a56b-c93b5c15e52d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_56656cb2-0db5-41a2-a56b-c93b5c15e52d\") failed: Property not found. retry in 1.254432008s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:21.706628157Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_56656cb2-0db5-41a2-a56b-c93b5c15e52d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_56656cb2-0db5-41a2-a56b-c93b5c15e52d\") failed: Property not found. retry in 1.237407544s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:22.971278124Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_56656cb2-0db5-41a2-a56b-c93b5c15e52d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:24.237466155Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_oximeter_2aadd0fc-5177-493a-9f7e-0593c0fe3187"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:24.237529344Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_crucible_pantry_36f087ee-bf6f-4c97-a999-ed642ab3cdab"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:24.237534256Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_crucible_f3a36f2d-8e26-42e8-92e1-a1d2b64e0b1c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:24.237538175Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_nexus_d0fa460e-c334-4a06-8890-451f7039247d"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:24.237541813Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_crucible_07d87730-3349-4204-8eec-fe9fdab9a6ba"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:24.237545526Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_nexus_a1b57c17-9dfb-443f-b708-833871354cad"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:24.237549576Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_crucible_79291adc-d002-4588-978b-5e78b8c0ba3c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:24.237573867Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_crucible_f05d7cc2-271e-4b70-9a2d-d625b0ea9fc7"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:24.23757817Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_crucible_8f0afcee-5e8e-470c-86a8-cb826d58e572"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:24.237581854Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_crucible_pantry_2b47746b-6454-483c-9ad4-332bf31d0843"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:24.237585285Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_clickhouse_56656cb2-0db5-41a2-a56b-c93b5c15e52d"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:24.237589082Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_external_dns_6e6e5549-d8aa-4b92-93c2-2b15052398a6"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:24.237592504Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_crucible_pantry_0656eb0b-5e08-42ef-aa7b-308375ba2f5f"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:24.237600915Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_crucible_cbb67d9f-d1a0-4a3c-8318-b095ffabde45"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:24.237604917Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_crucible_9ba6caf8-787f-4778-a0d1-8b2b954bfcf6"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:24.237608433Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_crucible_0acbec23-eeb8-49c8-b6f7-da12c2e314e7"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:24.237615427Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_external_dns_2591e26a-6588-4335-828d-4938ef1d6397"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:24.237782641Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_nexus_fe88893d-1075-499f-81ad-f02ad4112b77"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:24.237788632Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_crucible_91cd64e6-ebb3-41e8-b415-6061da85c422"} {"msg":"Writing ledger to /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:24.240574227Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:24.243455924Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:24.247207726Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"23c950c0-6b0f-4c29-b6cd-634dc8c6ab6b","remote_addr":"[fd00:1122:3344:101::1]:55030","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":43430928,"response_code":"204"} {"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:24.262422204Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1286"} {"msg":"Writing ledger to /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:24.262555446Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:24.262742259Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:24.265646537Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:687"} {"msg":"Nexus address: [fd00:1122:3344:101::b]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:24.267740281Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:700"} {"msg":"rack_network_config: RackNetworkConfigV2 {\n bfd: [],\n bgp: [],\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV2 {\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n autoneg: false,\n bgp_peers: [],\n lldp: None,\n port: \"qsfp0\",\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n local_pref: None,\n nexthop: 10.151.1.100,\n vlan_id: None,\n },\n ],\n switch: Switch0,\n uplink_port_fec: None,\n uplink_port_speed: Speed40G,\n },\n ],\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:24.305408107Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:867"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/1368d3f4-c0fd-4c61-a5b2-a8a3e63e09c5/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:24.30988282Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/1368d3f4-c0fd-4c61-a5b2-a8a3e63e09c5/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:24.420595329Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/1368d3f4-c0fd-4c61-a5b2-a8a3e63e09c5/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:24.502029029Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/1368d3f4-c0fd-4c61-a5b2-a8a3e63e09c5/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:24.669490206Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/1368d3f4-c0fd-4c61-a5b2-a8a3e63e09c5/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:24.868974216Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/1368d3f4-c0fd-4c61-a5b2-a8a3e63e09c5/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:25.100616701Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/1368d3f4-c0fd-4c61-a5b2-a8a3e63e09c5/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:25.36120614Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/1368d3f4-c0fd-4c61-a5b2-a8a3e63e09c5/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:25.60354264Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/1368d3f4-c0fd-4c61-a5b2-a8a3e63e09c5/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:25.918942485Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/1368d3f4-c0fd-4c61-a5b2-a8a3e63e09c5/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:26.506848992Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/1368d3f4-c0fd-4c61-a5b2-a8a3e63e09c5/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:27.17432831Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/1368d3f4-c0fd-4c61-a5b2-a8a3e63e09c5/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:28.088826094Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:28.318534668Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:59124"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:28.319199349Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"ec7dc208-e821-456f-a889-5c90e7a32605","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:59124","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":39,"response_code":"200"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/1368d3f4-c0fd-4c61-a5b2-a8a3e63e09c5/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:28.981638544Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/1368d3f4-c0fd-4c61-a5b2-a8a3e63e09c5/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:29.676056087Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/1368d3f4-c0fd-4c61-a5b2-a8a3e63e09c5/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:30.535780692Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:48:32.364649089Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Error Response: status: 503 Service Unavailable; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"60e84841-2605-4898-b326-1d8b6c2cdee5\\\", \\\"content-length\\\": \\\"133\\\", \\\"date\\\": \\\"Tue, 27 Aug 2024 19:48:32 GMT\\\"}; value: Error { error_code: Some(\\\"ServiceNotAvailable\\\"), message: \\\"Service Unavailable\\\", request_id: \\\"60e84841-2605-4898-b326-1d8b6c2cdee5\\\" }\"","delay":"44.938063464s"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d1398c6e-be69-4a31-a040-a709e17ad603\", \"content-length\": \"173\", \"date\": \"Tue, 27 Aug 2024 19:48:32 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"4b1d95e0-2253-4dc0-9406-d601ae11a340\\\"\", request_id: \"d1398c6e-be69-4a31-a040-a709e17ad603\" }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:33.133508693Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8ad6259c-c20d-484a-a3d3-f6f857c9b8f9\", \"content-length\": \"173\", \"date\": \"Tue, 27 Aug 2024 19:48:34 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"4b1d95e0-2253-4dc0-9406-d601ae11a340\\\"\", request_id: \"8ad6259c-c20d-484a-a3d3-f6f857c9b8f9\" }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:34.435743805Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"eea35c49-7d0e-4a26-940e-f28c6f15495e\", \"content-length\": \"173\", \"date\": \"Tue, 27 Aug 2024 19:48:37 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"4b1d95e0-2253-4dc0-9406-d601ae11a340\\\"\", request_id: \"eea35c49-7d0e-4a26-940e-f28c6f15495e\" }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:37.505760792Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5c63fc36-e8b7-4f2b-8823-e07d56271464\", \"content-length\": \"173\", \"date\": \"Tue, 27 Aug 2024 19:48:38 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"4b1d95e0-2253-4dc0-9406-d601ae11a340\\\"\", request_id: \"5c63fc36-e8b7-4f2b-8823-e07d56271464\" }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:39.400195843Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c51c3181-8268-4ead-a281-4e8a46ec60c9\", \"content-length\": \"173\", \"date\": \"Tue, 27 Aug 2024 19:48:42 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"4b1d95e0-2253-4dc0-9406-d601ae11a340\\\"\", request_id: \"c51c3181-8268-4ead-a281-4e8a46ec60c9\" }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:42.754047752Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0be555b3-cee0-4758-aa1b-385e110a5a50\", \"content-length\": \"173\", \"date\": \"Tue, 27 Aug 2024 19:48:46 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"4b1d95e0-2253-4dc0-9406-d601ae11a340\\\"\", request_id: \"0be555b3-cee0-4758-aa1b-385e110a5a50\" }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:46.512626266Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c8adc091-ce2a-4b01-ac1d-9a2506682c6f\", \"content-length\": \"173\", \"date\": \"Tue, 27 Aug 2024 19:48:49 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"4b1d95e0-2253-4dc0-9406-d601ae11a340\\\"\", request_id: \"c8adc091-ce2a-4b01-ac1d-9a2506682c6f\" }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:49.973494706Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"70544fea-bec1-4e32-bd7c-b38d04832af8\", \"content-length\": \"173\", \"date\": \"Tue, 27 Aug 2024 19:48:55 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"4b1d95e0-2253-4dc0-9406-d601ae11a340\\\"\", request_id: \"70544fea-bec1-4e32-bd7c-b38d04832af8\" }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:55.26981374Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:58.352493048Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:37009"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:48:58.353272311Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"6a0a0984-84b6-40a9-8286-a63ba3e3ec1a","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:37009","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":543,"response_code":"200"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"077245b9-e039-4de2-be96-c858b3894caf\", \"content-length\": \"173\", \"date\": \"Tue, 27 Aug 2024 19:49:00 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"4b1d95e0-2253-4dc0-9406-d601ae11a340\\\"\", request_id: \"077245b9-e039-4de2-be96-c858b3894caf\" }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:00.951970059Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c4b1027e-72db-4063-b960-8f2cacb0d92f\", \"content-length\": \"173\", \"date\": \"Tue, 27 Aug 2024 19:49:05 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"4b1d95e0-2253-4dc0-9406-d601ae11a340\\\"\", request_id: \"c4b1027e-72db-4063-b960-8f2cacb0d92f\" }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:05.347799176Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:14.173996966Z","hostname":"buskin","pid":1562,"component":"NexusNotifierTask","sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"sled-agent/src/nexus.rs:544","new_generation":"1","old_generation":"1"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:14.261300172Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:42656"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:14.263300275Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:14.350931237Z","hostname":"buskin","pid":1562,"uri":"/inventory","method":"GET","req_id":"ee715be0-031a-4ebb-9fd1-65ea2d7efe2e","remote_addr":"[fd00:1122:3344:101::a]:42656","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":89366,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:14.352896389Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:14.352980181Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:14.354405694Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:14.3548042Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"GET","req_id":"491e382b-2841-4cdf-8c23-a6b76bd47011","remote_addr":"[fd00:1122:3344:101::a]:42656","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":2753,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:16.248939344Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:45224"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:16.253001711Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:671","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 19de1a15-87dd-4421-b9c4-5a26a2f0078f, kind: Service { id: 6e6e5549-d8aa-4b92-93c2-2b15052398a6 }, name: Name(\"external-dns-6e6e5549-d8aa-4b92-93c2-2b15052398a6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 104])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 766b280f-b6d1-41b3-9cfc-b05604a1f00a, kind: Service { id: 2591e26a-6588-4335-828d-4938ef1d6397 }, name: Name(\"external-dns-2591e26a-6588-4335-828d-4938ef1d6397\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 172, 0])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1970a6ab-3c09-474a-bf05-c1dbea308c2c, kind: Service { id: a1b57c17-9dfb-443f-b708-833871354cad }, name: Name(\"nexus-a1b57c17-9dfb-443f-b708-833871354cad\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 182, 112])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: a4fc1828-cf68-4cee-9eba-aa2c2011d2a9, kind: Service { id: d0fa460e-c334-4a06-8890-451f7039247d }, name: Name(\"nexus-d0fa460e-c334-4a06-8890-451f7039247d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 221, 63])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e137c875-7fd4-407a-ba2a-2058e5219b83, kind: Service { id: fe88893d-1075-499f-81ad-f02ad4112b77 }, name: Name(\"nexus-fe88893d-1075-499f-81ad-f02ad4112b77\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 203, 34])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:16.253050392Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:16.253092632Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:16.2535693Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:16.25358973Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:16.25359756Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:16.253617731Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:16.253750193Z","hostname":"buskin","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"d9bdfa77-c703-422a-9524-851c4b77a378","remote_addr":"[fd00:1122:3344:101::b]:45224","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":4723,"response_code":"204"} {"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:16.255799936Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:939"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:16.447103643Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:38776"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:16.447205665Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:16.542206926Z","hostname":"buskin","pid":1562,"uri":"/inventory","method":"GET","req_id":"ed6d17e7-b7e5-408f-ab0f-e7227bde243f","remote_addr":"[fd00:1122:3344:101::b]:38776","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":94991,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:16.543235033Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:16.543299294Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:16.54365428Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:16.544076077Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"GET","req_id":"bb50d4f1-bc56-424d-a197-1da47eefc3c8","remote_addr":"[fd00:1122:3344:101::b]:38776","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1619,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:17.675338714Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:58257"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:17.67574463Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:671","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 19de1a15-87dd-4421-b9c4-5a26a2f0078f, kind: Service { id: 6e6e5549-d8aa-4b92-93c2-2b15052398a6 }, name: Name(\"external-dns-6e6e5549-d8aa-4b92-93c2-2b15052398a6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 104])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 766b280f-b6d1-41b3-9cfc-b05604a1f00a, kind: Service { id: 2591e26a-6588-4335-828d-4938ef1d6397 }, name: Name(\"external-dns-2591e26a-6588-4335-828d-4938ef1d6397\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 172, 0])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1970a6ab-3c09-474a-bf05-c1dbea308c2c, kind: Service { id: a1b57c17-9dfb-443f-b708-833871354cad }, name: Name(\"nexus-a1b57c17-9dfb-443f-b708-833871354cad\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 182, 112])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: a4fc1828-cf68-4cee-9eba-aa2c2011d2a9, kind: Service { id: d0fa460e-c334-4a06-8890-451f7039247d }, name: Name(\"nexus-d0fa460e-c334-4a06-8890-451f7039247d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 221, 63])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e137c875-7fd4-407a-ba2a-2058e5219b83, kind: Service { id: fe88893d-1075-499f-81ad-f02ad4112b77 }, name: Name(\"nexus-fe88893d-1075-499f-81ad-f02ad4112b77\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 203, 34])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:17.675800351Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:17.675832772Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:17.675850042Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:17.675856332Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:17.675862772Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:17.675878743Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:17.675910453Z","hostname":"buskin","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"a16da5c5-dfc8-49f2-953f-06fda38b7128","remote_addr":"[fd00:1122:3344:101::c]:58257","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":341,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:17.944551754Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:54105"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:17.944791137Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:671","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 19de1a15-87dd-4421-b9c4-5a26a2f0078f, kind: Service { id: 6e6e5549-d8aa-4b92-93c2-2b15052398a6 }, name: Name(\"external-dns-6e6e5549-d8aa-4b92-93c2-2b15052398a6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 104])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 766b280f-b6d1-41b3-9cfc-b05604a1f00a, kind: Service { id: 2591e26a-6588-4335-828d-4938ef1d6397 }, name: Name(\"external-dns-2591e26a-6588-4335-828d-4938ef1d6397\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 172, 0])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1970a6ab-3c09-474a-bf05-c1dbea308c2c, kind: Service { id: a1b57c17-9dfb-443f-b708-833871354cad }, name: Name(\"nexus-a1b57c17-9dfb-443f-b708-833871354cad\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 182, 112])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: a4fc1828-cf68-4cee-9eba-aa2c2011d2a9, kind: Service { id: d0fa460e-c334-4a06-8890-451f7039247d }, name: Name(\"nexus-d0fa460e-c334-4a06-8890-451f7039247d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 221, 63])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e137c875-7fd4-407a-ba2a-2058e5219b83, kind: Service { id: fe88893d-1075-499f-81ad-f02ad4112b77 }, name: Name(\"nexus-fe88893d-1075-499f-81ad-f02ad4112b77\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 203, 34])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:17.944849048Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:17.944876679Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:17.944892679Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:17.94492674Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:17.94494698Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:17.94495589Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:17.94497419Z","hostname":"buskin","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"a52a96e9-295f-4307-961d-8243e6eff59b","remote_addr":"[fd00:1122:3344:101::a]:54105","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":300,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:17.974970131Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:59267"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:17.975214715Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:671","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 19de1a15-87dd-4421-b9c4-5a26a2f0078f, kind: Service { id: 6e6e5549-d8aa-4b92-93c2-2b15052398a6 }, name: Name(\"external-dns-6e6e5549-d8aa-4b92-93c2-2b15052398a6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 104])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 766b280f-b6d1-41b3-9cfc-b05604a1f00a, kind: Service { id: 2591e26a-6588-4335-828d-4938ef1d6397 }, name: Name(\"external-dns-2591e26a-6588-4335-828d-4938ef1d6397\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 172, 0])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1970a6ab-3c09-474a-bf05-c1dbea308c2c, kind: Service { id: a1b57c17-9dfb-443f-b708-833871354cad }, name: Name(\"nexus-a1b57c17-9dfb-443f-b708-833871354cad\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 182, 112])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: a4fc1828-cf68-4cee-9eba-aa2c2011d2a9, kind: Service { id: d0fa460e-c334-4a06-8890-451f7039247d }, name: Name(\"nexus-d0fa460e-c334-4a06-8890-451f7039247d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 221, 63])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e137c875-7fd4-407a-ba2a-2058e5219b83, kind: Service { id: fe88893d-1075-499f-81ad-f02ad4112b77 }, name: Name(\"nexus-fe88893d-1075-499f-81ad-f02ad4112b77\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 203, 34])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:17.975254375Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:17.975287746Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:17.975297956Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:17.975310956Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:17.975317976Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:17.975324626Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:17.975345397Z","hostname":"buskin","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"fa07854f-cf15-4558-a8c7-9ad62d961aa5","remote_addr":"[fd00:1122:3344:101::b]:59267","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":226,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:23.606707265Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:39112"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:23.607110721Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"6c5e178d-ceb9-4da8-8c46-2f17e11197ce","remote_addr":"[fd00:1122:3344:101::b]:39112","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":387,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:23.674425301Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"a75528b0-e50e-47e7-af59-f568dde980b3","remote_addr":"[fd00:1122:3344:101::b]:39112","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":3210,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:28.385609033Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39423"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:28.385759645Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"1cbdf652-3ce6-49ff-bfdf-b84990d8c449","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39423","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":111,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:32.597640466Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:37342"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:32.598039463Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"3a68d7d4-7fa9-4c20-8615-665d07364f5f","remote_addr":"[fd00:1122:3344:101::b]:37342","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":382,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:32.599817892Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:33368"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:32.599919694Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"05e1356b-3863-4248-a3f1-69fe2946637e","remote_addr":"[fd00:1122:3344:101::b]:33368","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":35,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:32.608555005Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"d49cbd28-7306-4c1b-b841-e4fb0b984db8","remote_addr":"[fd00:1122:3344:101::b]:33368","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1453,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:32.627156219Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:44569"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:32.627287541Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"32c1600e-2d92-452d-b0ac-8fe37132fa3d","remote_addr":"[fd00:1122:3344:101::a]:44569","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":26,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:32.634748423Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"8215decb-5332-411f-a7f5-9d7a034986ea","remote_addr":"[fd00:1122:3344:101::a]:44569","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":61,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:32.722480417Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:51800"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:32.722559738Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"bf9861d9-4876-41e9-88db-6f3e587fec63","remote_addr":"[fd00:1122:3344:101::c]:51800","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":33,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:32.730390056Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"2871039a-8252-4935-9973-0a2e578cd56a","remote_addr":"[fd00:1122:3344:101::c]:51800","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":76,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:32.860865099Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:41475"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:32.861024589Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"f8893a0b-88f7-443f-8fb9-da2d7b731269","remote_addr":"[fd00:1122:3344:101::a]:41475","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":101,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:32.870208161Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:43065"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:32.872750553Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:32.977268941Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:33.042824972Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"5942c988-fc77-410e-b218-3d71790f5cd6","remote_addr":"[fd00:1122:3344:101::b]:43065","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":172551,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:33.043114047Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:58105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:33.043996741Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"5e08f96f-401e-433c-b4c3-6c32b551b705","remote_addr":"[fd00:1122:3344:101::c]:58105","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":822,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:33.049693234Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"edfc90c8-2575-4101-8cf0-803b3f256750","remote_addr":"[fd00:1122:3344:101::b]:43065","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":6366,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:33.061123631Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:36892"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:33.061257053Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:33.106651845Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:33.168877682Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"fdb65a1e-b5f0-473b-bc8d-4cf7338107d0","remote_addr":"[fd00:1122:3344:101::c]:36892","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":107677,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:33.169144596Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:51236"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:33.169486532Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"34b50e14-2aba-4b3f-a1b6-e1150136266a","remote_addr":"[fd00:1122:3344:101::c]:36892","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":201,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:33.170263785Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"Attempting to update network config with generation: 2, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:33.175552981Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:33.177101236Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:33.178905596Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 2","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:33.180635454Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:33.180784037Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"PUT","req_id":"306119c6-0b40-4efb-9c63-01c9be10e609","remote_addr":"[fd00:1122:3344:101::b]:43065","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":11491,"response_code":"204"} {"msg":"Attempting to update network config with generation: 3, current_generation: 2","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:33.181506679Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/1920ef9b-1f2c-4f76-9da0-39371846beaa/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:33.181532349Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/d862b57c-61f8-46cc-abb0-f7417780b9ed/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:33.181710782Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 3","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:33.181854004Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:33.181883294Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"PUT","req_id":"999fc694-0c10-4c8d-a492-8bc0298cc45d","remote_addr":"[fd00:1122:3344:101::c]:36892","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":526,"response_code":"204"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:33.210274949Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:33.262538433Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"33e5e817-d52e-4e4c-b478-8b7818b724e0","remote_addr":"[fd00:1122:3344:101::a]:51236","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":93155,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:33.263051881Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"4601d5d7-5540-4688-a543-c82d88cbec7c","remote_addr":"[fd00:1122:3344:101::a]:51236","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":167,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:47.434736393Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:46199","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","component":"producer-server","sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::d]:59584"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:47.485918132Z","hostname":"buskin","pid":1562,"uri":"/4b1d95e0-2253-4dc0-9406-d601ae11a340","method":"GET","req_id":"e2e3b2a6-56ef-4967-9239-81e2fd919722","remote_addr":"[fd00:1122:3344:101::d]:59584","local_addr":"[fd00:1122:3344:101::1]:46199","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":50837,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:58.418484472Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:41451"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:49:58.419053241Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"e2d14d79-e6e0-440f-b0a7-3ad3694ca172","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:41451","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":58,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:02.287944088Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:43084"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:02.288764761Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"9c47fd0e-3146-483d-8c13-e9e2532102e7","remote_addr":"[fd00:1122:3344:101::b]:43084","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":783,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:02.296714821Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"1b0b855a-5b1a-4e46-ac47-08c28f85355b","remote_addr":"[fd00:1122:3344:101::b]:43084","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":140,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:02.324699128Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:39184"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:02.324870691Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"af7acaba-ac6b-4ff8-856b-625b2d4efc0d","remote_addr":"[fd00:1122:3344:101::b]:39184","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":132,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:02.470972669Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:61491"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:02.472025416Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"77348a93-9a4a-4ca2-b7c0-c6d730e2295e","remote_addr":"[fd00:1122:3344:101::a]:61491","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1023,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:02.478288398Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"6c135cb7-8690-44ce-b0d2-c1a059a1caec","remote_addr":"[fd00:1122:3344:101::a]:61491","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":102,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:02.488091508Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:42227"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:02.488187389Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"0e81fde5-ceeb-40aa-87f2-fb771aaac7b7","remote_addr":"[fd00:1122:3344:101::c]:42227","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":30,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:02.495272895Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"6d1f0821-68f2-44aa-a27b-c172a65fbda9","remote_addr":"[fd00:1122:3344:101::c]:42227","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":87,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:02.573013736Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:44352"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:02.573107647Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:02.611888561Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:02.668703919Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"161ac403-26e2-4624-8d29-723928bddca1","remote_addr":"[fd00:1122:3344:101::b]:44352","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":95654,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:02.668859792Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:65337"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:02.668921993Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:50161"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:02.669119526Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"5814a2db-f71c-41a8-8e07-7cd55f1c3443","remote_addr":"[fd00:1122:3344:101::c]:65337","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":197,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:02.669226298Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"d1d729b7-96d4-41d5-94df-8c6fff6bf488","remote_addr":"[fd00:1122:3344:101::a]:50161","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":215,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:02.669317289Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"a317ff0c-3d1f-4985-b35e-4eaf91138d49","remote_addr":"[fd00:1122:3344:101::b]:44352","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":301,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:02.678373888Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:48219"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:02.67852782Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:02.715058807Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:02.769279103Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"7e9262a4-6af0-46b3-88e9-3cfe3c56f85b","remote_addr":"[fd00:1122:3344:101::c]:48219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":90827,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:02.769538238Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:40444"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:02.769748001Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:02.769778402Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"20583ea4-ab6b-450d-aa43-28112275d054","remote_addr":"[fd00:1122:3344:101::c]:48219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":211,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:02.805616217Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:02.860779089Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"177eb19f-f812-4cff-9b09-3c65187b4be2","remote_addr":"[fd00:1122:3344:101::a]:40444","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":91129,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:02.861094214Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"75992d1a-560e-4122-a16b-58cbb9eedf0d","remote_addr":"[fd00:1122:3344:101::a]:40444","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":127,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:17.447962769Z","hostname":"buskin","pid":1562,"uri":"/4b1d95e0-2253-4dc0-9406-d601ae11a340","method":"GET","req_id":"e777d2d3-e4b8-4950-ab5d-c850f50376b0","remote_addr":"[fd00:1122:3344:101::d]:59584","local_addr":"[fd00:1122:3344:101::1]:46199","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":7742,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:28.46506795Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:64581"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:28.465170051Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"7fd713cf-4cc9-4dcf-a259-31a32fc0586c","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:64581","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":92,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:32.367109252Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:52922"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:32.368609546Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"4ef0ee43-7b48-421f-8e20-49ba7a8bff34","remote_addr":"[fd00:1122:3344:101::b]:52922","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1480,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:32.389221094Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"5ccdb951-b405-4efe-96b7-88cf25eb17ba","remote_addr":"[fd00:1122:3344:101::b]:52922","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":116,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:32.453584996Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:43297"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:32.45387669Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"8085e5dd-90a3-4a44-8eeb-2984c4024088","remote_addr":"[fd00:1122:3344:101::b]:43297","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":226,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:32.648713294Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:65066"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:32.648835996Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:32.690471007Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:32.751145788Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"ee05b4b5-de58-4ee7-84b8-8ac253c44f35","remote_addr":"[fd00:1122:3344:101::b]:65066","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":102362,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:32.751339941Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:57524"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:32.751486644Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"e5da7fb8-5472-4d20-9451-3595059dd173","remote_addr":"[fd00:1122:3344:101::c]:57524","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":105,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:32.751662757Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"6d07b6d5-f947-468b-919c-2f394ca58ea0","remote_addr":"[fd00:1122:3344:101::b]:65066","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":211,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:32.758825973Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"11d22280-1cb5-462e-abec-94e729a216aa","remote_addr":"[fd00:1122:3344:101::c]:57524","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":115,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:32.769903385Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:33930"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:32.769981386Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"9904454c-9876-41c9-9859-667f85603b47","remote_addr":"[fd00:1122:3344:101::a]:33930","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":27,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:32.777111032Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"7ec1da94-3940-4960-8c8f-f5f82bb60a1b","remote_addr":"[fd00:1122:3344:101::a]:33930","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":866,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:32.803081267Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:52040"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:32.803247969Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"41ac6056-d3c4-487c-b0fd-df03e714263f","remote_addr":"[fd00:1122:3344:101::c]:52040","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":115,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:32.812959728Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:55288"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:32.81306899Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"0b62277c-dbe2-454d-bd9b-5ef08be37112","remote_addr":"[fd00:1122:3344:101::a]:55288","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":74,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:33.032357513Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:57561"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:33.032456805Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:33.069430259Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:33.125066518Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"03ce85b9-ea4b-4512-88c6-823c73277d71","remote_addr":"[fd00:1122:3344:101::c]:57561","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":92638,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:33.1252053Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:48038"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:33.125373403Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:33.129334388Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"27817aac-b2bf-4917-9411-1f76b64dd68e","remote_addr":"[fd00:1122:3344:101::c]:57561","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":3889,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:33.15884249Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:33.21392245Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"9c4b80a9-836c-40f0-b505-c045a6343a8b","remote_addr":"[fd00:1122:3344:101::a]:48038","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":88617,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:33.214292826Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"20fd1030-9a83-4a4a-9758-f4df96f7a9e7","remote_addr":"[fd00:1122:3344:101::a]:48038","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":165,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:47.258358081Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:47.449496826Z","hostname":"buskin","pid":1562,"uri":"/4b1d95e0-2253-4dc0-9406-d601ae11a340","method":"GET","req_id":"b836c966-2dc5-487c-99b7-688244738b9b","remote_addr":"[fd00:1122:3344:101::d]:59584","local_addr":"[fd00:1122:3344:101::1]:46199","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":6572,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:58.499153515Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:47640"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:50:58.499972908Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"c7f56597-f184-4e00-91b9-4d4b5b928439","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:47640","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":802,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:02.34100152Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:64606"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:02.341147082Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"24dcac72-fe95-4c4c-bec7-c9fef1543b2b","remote_addr":"[fd00:1122:3344:101::b]:64606","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":126,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:02.349661081Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:32809"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:02.349861534Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"f1bd81b1-83b4-4ce8-9795-16555e4b5239","remote_addr":"[fd00:1122:3344:101::b]:32809","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":139,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:02.354875236Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"cc716704-5a42-4e47-964d-17c426be8137","remote_addr":"[fd00:1122:3344:101::b]:64606","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":129,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:02.491971786Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:63978"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:02.492077278Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"45bf83fa-19dc-4a43-b72c-c6fb9f804a9f","remote_addr":"[fd00:1122:3344:101::a]:63978","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":54,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:02.499009471Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:37407"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:02.499179634Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:02.504255557Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"42b398d5-58e4-4465-808c-a5e8ee3b9a4c","remote_addr":"[fd00:1122:3344:101::a]:63978","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":4964,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:02.522255691Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:43559"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:02.522412554Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"40487cb1-4d46-4e6f-92fd-4bb43945167d","remote_addr":"[fd00:1122:3344:101::c]:43559","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":103,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:02.531700215Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"20eba95b-bbb1-46b7-ba61-5646b8388273","remote_addr":"[fd00:1122:3344:101::c]:43559","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1277,"response_code":"204"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:02.542572313Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:02.561854758Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:56081"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:02.562082672Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"5a73ad4b-d3ae-499e-95b5-06ceb8b56729","remote_addr":"[fd00:1122:3344:101::c]:56081","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":182,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:02.610731207Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"47d363ef-8c58-4fb3-97f3-d6ffed2eb897","remote_addr":"[fd00:1122:3344:101::b]:37407","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":111611,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:02.611132273Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"12861a50-3f6c-4845-a0f2-bcf490e4ae76","remote_addr":"[fd00:1122:3344:101::b]:37407","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":143,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:02.61585598Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:34218"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:02.616022533Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"1b4e8bb3-b8f8-4c43-96ee-80b042397352","remote_addr":"[fd00:1122:3344:101::a]:34218","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":133,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:02.716950812Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:63463"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:02.717921728Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:02.75352729Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:02.811716311Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"9fb3b785-002a-43d9-843e-697f7d524286","remote_addr":"[fd00:1122:3344:101::a]:63463","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":94679,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:02.811909404Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:46670"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:02.812195659Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"64f57c7e-9b53-4c4a-a472-c4c15fda239f","remote_addr":"[fd00:1122:3344:101::a]:63463","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":187,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:02.812234619Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:02.845836128Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:02.896465336Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"2431a4e6-5581-4861-876b-7421cd1af304","remote_addr":"[fd00:1122:3344:101::c]:46670","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":84348,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:02.896791741Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"8dfa4cff-a888-443a-a11d-9c868cf4a329","remote_addr":"[fd00:1122:3344:101::c]:46670","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":130,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:17.475902093Z","hostname":"buskin","pid":1562,"uri":"/4b1d95e0-2253-4dc0-9406-d601ae11a340","method":"GET","req_id":"83c13787-deee-4535-bb1d-7ce8eb819a27","remote_addr":"[fd00:1122:3344:101::d]:59584","local_addr":"[fd00:1122:3344:101::1]:46199","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":6655,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:28.582079334Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:64510"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:28.583343948Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"bc52eaf3-1961-4884-a684-5c69f32aab1c","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:64510","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1248,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:32.409235934Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:51006"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:32.409326735Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"f20e0dae-1296-40d9-a9ee-65acfb45754d","remote_addr":"[fd00:1122:3344:101::b]:51006","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":56,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:32.426769201Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"f615f1db-436f-435d-92a8-07d01f936d83","remote_addr":"[fd00:1122:3344:101::b]:51006","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":124,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:32.497798804Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:38144"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:32.497994337Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"abe4088b-9205-49e1-a0f5-f03419342678","remote_addr":"[fd00:1122:3344:101::b]:38144","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":162,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:32.677180582Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:39931"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:32.677287013Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"388ad862-256a-449a-bb91-8d499cb34a56","remote_addr":"[fd00:1122:3344:101::c]:39931","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":49,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:32.684487054Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"1cd71d52-84c0-4045-9872-79e5bbbea719","remote_addr":"[fd00:1122:3344:101::c]:39931","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":99,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:32.688770292Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:41154"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:32.688943914Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"fa7904a4-5340-4ad6-9fab-53d71f9f06cb","remote_addr":"[fd00:1122:3344:101::a]:41154","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":98,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:32.698592422Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:39716"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:32.698720784Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"3dccedef-9f2f-45ed-8986-d5e7567d242a","remote_addr":"[fd00:1122:3344:101::c]:39716","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":83,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:32.78637017Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:36768"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:32.786557293Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:32.825024493Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:32.881942155Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"b24d3619-9344-4c3a-b2c1-59cbbf79f788","remote_addr":"[fd00:1122:3344:101::b]:36768","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":95663,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:32.882192939Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:35402"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:32.882413763Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"ab9b0ba5-5fc3-4229-b870-6babeecb4a01","remote_addr":"[fd00:1122:3344:101::a]:35402","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":129,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:32.883360598Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"3788e5c2-77bb-4b8f-a581-65209fe25327","remote_addr":"[fd00:1122:3344:101::b]:36768","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1032,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:32.890340082Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"a5082fa4-229f-43e5-8074-c8e35d2d4ae9","remote_addr":"[fd00:1122:3344:101::a]:35402","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":121,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:32.958894615Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:59090"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:32.959606267Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:32.99483699Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:33.057758984Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"a3ca73a5-efba-4472-bc75-f5d5cb9fc156","remote_addr":"[fd00:1122:3344:101::a]:59090","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":98987,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:33.057903076Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:36298"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:33.058008778Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:33.09294663Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:33.145018362Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"fedf6bf4-1e52-45f2-bf0f-f185cca33d18","remote_addr":"[fd00:1122:3344:101::c]:36298","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":87161,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:33.145328947Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"13caee34-2e3f-4f5b-bb4e-9bde5e820a93","remote_addr":"[fd00:1122:3344:101::a]:59090","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":139,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:33.145488299Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"d2d4eb48-a37b-46f2-9991-601ba203b5f1","remote_addr":"[fd00:1122:3344:101::c]:36298","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":220,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:47.524446149Z","hostname":"buskin","pid":1562,"uri":"/4b1d95e0-2253-4dc0-9406-d601ae11a340","method":"GET","req_id":"e26f23ae-cd2c-4f0e-b834-262d90ce38b4","remote_addr":"[fd00:1122:3344:101::d]:59584","local_addr":"[fd00:1122:3344:101::1]:46199","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":8128,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:58.605891919Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:58634"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:51:58.605952854Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"5cfa3591-79f3-4f1e-8a7a-9fefa0b199ee","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:58634","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":38,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:02.375128234Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:57673"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:02.375212546Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"546d4cbe-62ea-47cd-9bad-395b75cb8356","remote_addr":"[fd00:1122:3344:101::b]:57673","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":63,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:02.380009265Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:47445"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:02.380204528Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"01a27644-3777-44b7-8d13-4d4653841946","remote_addr":"[fd00:1122:3344:101::b]:47445","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":140,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:02.385809169Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"5954bd05-b71c-4d77-b11f-40bf750c15eb","remote_addr":"[fd00:1122:3344:101::b]:57673","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":106,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:02.502423451Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:55342"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:02.502553182Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"fb656798-849b-4e73-8aca-870a9663afb6","remote_addr":"[fd00:1122:3344:101::a]:55342","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":113,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:02.510446202Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"888197b7-bb52-43af-9237-d205a9655ab7","remote_addr":"[fd00:1122:3344:101::a]:55342","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":98,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:02.530286726Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:57738"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:02.53035295Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:02.568162587Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:02.624279907Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"e9d5addc-e389-4503-a886-36d7e5839476","remote_addr":"[fd00:1122:3344:101::b]:57738","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":94213,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:02.62446923Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:49212"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:02.62448395Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:34616"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:02.624529721Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:53009"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:02.624767175Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"f1ee20ce-b847-4098-961f-df58a9c1f6ed","remote_addr":"[fd00:1122:3344:101::b]:57738","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":208,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:02.624896397Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"d4e3d666-8b32-4a01-822b-70c4dcaee52a","remote_addr":"[fd00:1122:3344:101::c]:49212","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":295,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:02.624916397Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"68056d66-6f9d-4dc2-83e5-bdb91fc1439a","remote_addr":"[fd00:1122:3344:101::a]:53009","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":264,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:02.624933387Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"5126908d-ed3d-49bb-8cb2-dcd9c22f3caa","remote_addr":"[fd00:1122:3344:101::c]:34616","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":184,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:02.6336577Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"5996541d-a1bd-4808-a4d3-258656ad0b6b","remote_addr":"[fd00:1122:3344:101::c]:34616","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":801,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:02.659822259Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:41830"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:02.65990051Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:02.695210779Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:02.749322796Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"7d7ca048-bc6f-410a-ac08-3d23f254a35f","remote_addr":"[fd00:1122:3344:101::a]:41830","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":89697,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:02.749627041Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"ddb79923-2741-4237-9c28-6fd83c45c752","remote_addr":"[fd00:1122:3344:101::a]:41830","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":117,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:02.819227521Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:52134"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:02.819397584Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:02.859257917Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:02.915663962Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"82d6c2fc-588c-4717-a84c-c435da98201c","remote_addr":"[fd00:1122:3344:101::c]:52134","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":96599,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:02.91618739Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"da8082ed-8e76-4e82-8c49-87c7d78e1315","remote_addr":"[fd00:1122:3344:101::c]:52134","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":240,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:17.533688903Z","hostname":"buskin","pid":1562,"uri":"/4b1d95e0-2253-4dc0-9406-d601ae11a340","method":"GET","req_id":"8a8e0b40-3e8a-4510-9965-361189a51dc6","remote_addr":"[fd00:1122:3344:101::d]:59584","local_addr":"[fd00:1122:3344:101::1]:46199","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":6373,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:28.607651958Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:53924"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:28.608495902Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"c0fb8d57-006b-4f81-aa15-3191cd5a94b8","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:53924","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":32,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:32.523594216Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:40688"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:32.523788479Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"ad8836a2-984a-4d8f-984b-36c9d5c9a7ee","remote_addr":"[fd00:1122:3344:101::b]:40688","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":182,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:32.584869047Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:58890"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:32.585005089Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"a02aea93-abba-47a3-a787-dcb9e11bb64f","remote_addr":"[fd00:1122:3344:101::a]:58890","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":101,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:32.628750564Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:53806"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:32.628842776Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"445f56a2-52c7-4a50-9bc2-04b8d167dbb3","remote_addr":"[fd00:1122:3344:101::b]:53806","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":49,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:32.636238467Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"d02432ed-d96e-4340-8ae9-b14f4236f981","remote_addr":"[fd00:1122:3344:101::b]:53806","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":98,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:32.675252434Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:35115"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:32.675329606Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"a8544aa9-aa93-49e2-9203-f82873cd69d6","remote_addr":"[fd00:1122:3344:101::a]:35115","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":28,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:32.68168798Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"d4c1f3b4-3e19-4713-ac23-e3036794ca3b","remote_addr":"[fd00:1122:3344:101::a]:35115","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":74,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:32.704626975Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:58503"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:32.704778047Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"73f3be65-e928-43bb-85a3-181630e54e11","remote_addr":"[fd00:1122:3344:101::c]:58503","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":89,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:32.726569643Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:36196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:32.726649744Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"ac730e35-58dd-409a-bbfb-5ef7b0a274aa","remote_addr":"[fd00:1122:3344:101::c]:36196","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":39,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:32.734649905Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"949f1aec-97cc-4459-bee1-3d854347f497","remote_addr":"[fd00:1122:3344:101::c]:36196","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":89,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:32.815201162Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:34774"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:32.815839722Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:32.853510288Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:32.904747915Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"2d7a15c1-b66f-4976-8d99-b30753c1008e","remote_addr":"[fd00:1122:3344:101::a]:34774","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":89504,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:32.904992619Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:64849"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:32.905079051Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:36609"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:32.905220973Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:32.905345735Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"bc217fc9-bb24-4caf-a042-2b1652a03b16","remote_addr":"[fd00:1122:3344:101::a]:34774","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":277,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:32.938347744Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:32.984687701Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:32.984738602Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"a2b8a434-ae77-4084-8ff2-19ccba74dd40","remote_addr":"[fd00:1122:3344:101::c]:64849","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":79592,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:32.987985525Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"0dbf2239-3221-4287-bd69-bfbb001cbb53","remote_addr":"[fd00:1122:3344:101::c]:64849","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":561,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:33.016695685Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:33.065859618Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"3d6f2bd0-bfad-4c17-8c0f-3fc5c4e6e705","remote_addr":"[fd00:1122:3344:101::b]:36609","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":160640,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:33.066229854Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"f6fed444-32e3-4b3b-95d9-6ae8c8221fe5","remote_addr":"[fd00:1122:3344:101::b]:36609","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":107,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:47.52980341Z","hostname":"buskin","pid":1562,"uri":"/4b1d95e0-2253-4dc0-9406-d601ae11a340","method":"GET","req_id":"fb54c64c-c1e9-49b4-b2ea-6f6522321eb2","remote_addr":"[fd00:1122:3344:101::d]:59584","local_addr":"[fd00:1122:3344:101::1]:46199","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":10557,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:58.643022147Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:60241"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:52:58.643099999Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"301591d6-29de-4e09-9b78-3809ec5e03e9","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:60241","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":45,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:02.333923868Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:36596"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:02.334005419Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"41df1531-cab2-48eb-afdc-31341e577a78","remote_addr":"[fd00:1122:3344:101::b]:36596","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":60,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:02.342714271Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"db1e1a7b-3921-47b8-a31f-1c497c7955ae","remote_addr":"[fd00:1122:3344:101::b]:36596","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1052,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:02.353237794Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:47821"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:02.353409766Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"65ccafc8-8622-47bc-9e16-21bf21546fc0","remote_addr":"[fd00:1122:3344:101::b]:47821","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":126,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:02.495928645Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:61576"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:02.496012146Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"8b8d184e-f1df-4b0b-a5ed-97f058e11826","remote_addr":"[fd00:1122:3344:101::a]:61576","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":43,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:02.502923519Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"5ca8362c-727f-43ee-ab05-a272779776d2","remote_addr":"[fd00:1122:3344:101::a]:61576","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":98,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:02.518044476Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:37948"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:02.518169008Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"77693d5f-b27d-4038-b735-14acef556912","remote_addr":"[fd00:1122:3344:101::a]:37948","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":102,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:02.533251325Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:39472"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:02.533329626Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"bb8c1cd1-4b52-4286-867b-4d61adf2381d","remote_addr":"[fd00:1122:3344:101::c]:39472","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":27,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:02.539983955Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"88630235-a314-42ca-b532-72e20a79cb86","remote_addr":"[fd00:1122:3344:101::c]:39472","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":55,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:02.56052793Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:38605"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:02.560603442Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:02.600151598Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:02.658238417Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"50498b78-400b-4ea3-a893-618367e86654","remote_addr":"[fd00:1122:3344:101::b]:38605","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":97670,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:02.658475281Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:40997"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:02.658674885Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"125aba94-d170-4b0d-b6d2-0e32ee6448d5","remote_addr":"[fd00:1122:3344:101::c]:40997","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":165,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:02.658687985Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"66dab315-3868-4e1e-9f7a-44f71177f119","remote_addr":"[fd00:1122:3344:101::b]:38605","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":166,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:02.71652577Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:40918"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:02.717353204Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:02.753609226Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:02.809282096Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"1f906571-136e-46ed-9a69-6c720efc6592","remote_addr":"[fd00:1122:3344:101::c]:40918","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":92674,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:02.80956586Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:52478"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:02.809755094Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"ce7e55fd-b283-4ec0-8924-4d450020ea81","remote_addr":"[fd00:1122:3344:101::c]:40918","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":211,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:02.810471735Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:02.847205106Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:02.903616598Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"76f97659-c6d4-4281-a29b-18f04a3ef193","remote_addr":"[fd00:1122:3344:101::a]:52478","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":93937,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:02.903979403Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"bfc803e8-0e7c-4475-8bf9-3274ad9f6cb6","remote_addr":"[fd00:1122:3344:101::a]:52478","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":138,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:17.525137925Z","hostname":"buskin","pid":1562,"uri":"/4b1d95e0-2253-4dc0-9406-d601ae11a340","method":"GET","req_id":"08d09d62-0c8c-43bc-9737-db948fafa8d8","remote_addr":"[fd00:1122:3344:101::d]:59584","local_addr":"[fd00:1122:3344:101::1]:46199","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":6622,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:28.676184009Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:53531"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:28.676270251Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"bf1ae170-5f92-4cf6-821d-6bd4a95b677d","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:53531","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":52,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.388685367Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:57329"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.389996948Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"f9f5ad31-1f2c-40e3-a40c-b6646949ad34","remote_addr":"[fd00:1122:3344:101::b]:57329","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1280,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.397166125Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"ac44b8db-7ecf-44e7-ae78-a0d5bd19af92","remote_addr":"[fd00:1122:3344:101::b]:57329","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":124,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.451735948Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:55622"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.452010852Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"9e8833f7-0eb8-4c4e-a228-4554fb4d7c79","remote_addr":"[fd00:1122:3344:101::b]:55622","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":241,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.703737785Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:41120"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.703960829Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:671","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 19de1a15-87dd-4421-b9c4-5a26a2f0078f, kind: Service { id: 6e6e5549-d8aa-4b92-93c2-2b15052398a6 }, name: Name(\"external-dns-6e6e5549-d8aa-4b92-93c2-2b15052398a6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 104])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 766b280f-b6d1-41b3-9cfc-b05604a1f00a, kind: Service { id: 2591e26a-6588-4335-828d-4938ef1d6397 }, name: Name(\"external-dns-2591e26a-6588-4335-828d-4938ef1d6397\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 172, 0])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1970a6ab-3c09-474a-bf05-c1dbea308c2c, kind: Service { id: a1b57c17-9dfb-443f-b708-833871354cad }, name: Name(\"nexus-a1b57c17-9dfb-443f-b708-833871354cad\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 182, 112])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: a4fc1828-cf68-4cee-9eba-aa2c2011d2a9, kind: Service { id: d0fa460e-c334-4a06-8890-451f7039247d }, name: Name(\"nexus-d0fa460e-c334-4a06-8890-451f7039247d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 221, 63])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e137c875-7fd4-407a-ba2a-2058e5219b83, kind: Service { id: fe88893d-1075-499f-81ad-f02ad4112b77 }, name: Name(\"nexus-fe88893d-1075-499f-81ad-f02ad4112b77\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 203, 34])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.70402566Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.70404972Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.704067731Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.704091071Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.704104661Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.704136032Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.704158392Z","hostname":"buskin","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"c739974a-5c7d-46c3-a1ef-6f6b730bcb71","remote_addr":"[fd00:1122:3344:101::b]:41120","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":381,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.730482266Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:42745"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.730592164Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"f5f36551-9c8f-4202-a3c9-f8b541b4c434","remote_addr":"[fd00:1122:3344:101::a]:42745","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":79,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.733907799Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:38548"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.734062001Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"6007f5b6-904a-410c-9f2a-5b474b612af9","remote_addr":"[fd00:1122:3344:101::c]:38548","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":70,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.743450795Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:53558"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.743514225Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"b6941d85-17b9-4659-b33c-0f0ff6ce2d1d","remote_addr":"[fd00:1122:3344:101::a]:53558","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":38,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.747664393Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:48013"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.748716941Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:671","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 19de1a15-87dd-4421-b9c4-5a26a2f0078f, kind: Service { id: 6e6e5549-d8aa-4b92-93c2-2b15052398a6 }, name: Name(\"external-dns-6e6e5549-d8aa-4b92-93c2-2b15052398a6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 104])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 766b280f-b6d1-41b3-9cfc-b05604a1f00a, kind: Service { id: 2591e26a-6588-4335-828d-4938ef1d6397 }, name: Name(\"external-dns-2591e26a-6588-4335-828d-4938ef1d6397\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 172, 0])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1970a6ab-3c09-474a-bf05-c1dbea308c2c, kind: Service { id: a1b57c17-9dfb-443f-b708-833871354cad }, name: Name(\"nexus-a1b57c17-9dfb-443f-b708-833871354cad\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 182, 112])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: a4fc1828-cf68-4cee-9eba-aa2c2011d2a9, kind: Service { id: d0fa460e-c334-4a06-8890-451f7039247d }, name: Name(\"nexus-d0fa460e-c334-4a06-8890-451f7039247d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 221, 63])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e137c875-7fd4-407a-ba2a-2058e5219b83, kind: Service { id: fe88893d-1075-499f-81ad-f02ad4112b77 }, name: Name(\"nexus-fe88893d-1075-499f-81ad-f02ad4112b77\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 203, 34])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.748741521Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.748775182Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.748795122Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.748802822Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.748837023Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.748855193Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.748866753Z","hostname":"buskin","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"ba66ef70-1e96-4beb-8c93-99cf933f80bb","remote_addr":"[fd00:1122:3344:101::a]:48013","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1168,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.750868006Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"bba2053b-38c2-4a81-8cec-ce9de9693958","remote_addr":"[fd00:1122:3344:101::a]:53558","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":59,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.772500579Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:37593"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.773410194Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.811941384Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.869605206Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"7240d84d-8f02-467a-88ef-d2f03f4efdab","remote_addr":"[fd00:1122:3344:101::b]:37593","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":97061,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.869767049Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:33701"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.86985509Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"44bf619a-869c-49a0-ad63-2184e327db14","remote_addr":"[fd00:1122:3344:101::c]:33701","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":46,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.870044873Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"c2979492-23e2-4f81-885d-4a3f1b2b55c4","remote_addr":"[fd00:1122:3344:101::b]:37593","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":122,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.875363271Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:34867"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.87654874Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:671","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 19de1a15-87dd-4421-b9c4-5a26a2f0078f, kind: Service { id: 6e6e5549-d8aa-4b92-93c2-2b15052398a6 }, name: Name(\"external-dns-6e6e5549-d8aa-4b92-93c2-2b15052398a6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 104])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 766b280f-b6d1-41b3-9cfc-b05604a1f00a, kind: Service { id: 2591e26a-6588-4335-828d-4938ef1d6397 }, name: Name(\"external-dns-2591e26a-6588-4335-828d-4938ef1d6397\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 172, 0])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1970a6ab-3c09-474a-bf05-c1dbea308c2c, kind: Service { id: a1b57c17-9dfb-443f-b708-833871354cad }, name: Name(\"nexus-a1b57c17-9dfb-443f-b708-833871354cad\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 182, 112])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: a4fc1828-cf68-4cee-9eba-aa2c2011d2a9, kind: Service { id: d0fa460e-c334-4a06-8890-451f7039247d }, name: Name(\"nexus-d0fa460e-c334-4a06-8890-451f7039247d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 221, 63])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e137c875-7fd4-407a-ba2a-2058e5219b83, kind: Service { id: fe88893d-1075-499f-81ad-f02ad4112b77 }, name: Name(\"nexus-fe88893d-1075-499f-81ad-f02ad4112b77\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 203, 34])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.876607071Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.876654672Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.876686302Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.876715893Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.876746833Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.876779504Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.876811094Z","hostname":"buskin","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"fc05f754-2b6c-4915-bd8c-d900374c15af","remote_addr":"[fd00:1122:3344:101::c]:34867","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":516,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.876958186Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"77c7e579-052b-4173-b294-65741701044a","remote_addr":"[fd00:1122:3344:101::c]:33701","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":78,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.90900514Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:48492"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.909093412Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:32.946010745Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:33.00317462Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"be2fe1f9-e467-4868-86f9-ea3fbf27810e","remote_addr":"[fd00:1122:3344:101::a]:48492","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":94124,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:33.003465384Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:61037"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:33.003692238Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"07270704-0245-45d8-80ea-0a063ed81e5e","remote_addr":"[fd00:1122:3344:101::a]:48492","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":232,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:33.004354929Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:33.04482007Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:33.099806689Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"3d626a2f-5864-4a44-9532-7ad5c39d8ac1","remote_addr":"[fd00:1122:3344:101::c]:61037","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":96216,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:33.100123644Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"0e8f5b78-540e-49ce-ae8a-d2c963df112a","remote_addr":"[fd00:1122:3344:101::c]:61037","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":139,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:47.525246962Z","hostname":"buskin","pid":1562,"uri":"/4b1d95e0-2253-4dc0-9406-d601ae11a340","method":"GET","req_id":"85ef19fb-666c-42ee-aa16-84cc3f931ded","remote_addr":"[fd00:1122:3344:101::d]:59584","local_addr":"[fd00:1122:3344:101::1]:46199","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":6793,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:58.707612104Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:48124"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:53:58.707678365Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"3bb191fa-d03e-4fbb-ba88-7f81df0d9a3e","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:48124","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":51,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:02.324751476Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:64465"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:02.324869018Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"056c6ed4-6aa8-407e-b4da-6be99ebdc6ed","remote_addr":"[fd00:1122:3344:101::b]:64465","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":103,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:02.33236771Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"502255a4-3edc-4a8d-9cfc-b24c55806025","remote_addr":"[fd00:1122:3344:101::b]:64465","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":697,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:02.412573671Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:54930"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:02.412775584Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"2b0811ea-43d1-46ad-a7cb-bf1e8abcff7a","remote_addr":"[fd00:1122:3344:101::b]:54930","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":169,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:02.478400077Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:56291"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:02.478486778Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"96ee9a14-3f88-4449-bb57-b2dfc1a1ec37","remote_addr":"[fd00:1122:3344:101::a]:56291","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":64,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:02.484964004Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"24ae0abf-1102-4f9c-975e-f38ef20e5c25","remote_addr":"[fd00:1122:3344:101::a]:56291","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":88,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:02.515793137Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:63570"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:02.516773653Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"74d1e2b1-d115-494a-8d81-0e82055dfed4","remote_addr":"[fd00:1122:3344:101::a]:63570","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":942,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:02.551748695Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:60558"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:02.551834806Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"f214fb47-97a3-4925-926d-8f393d4722f6","remote_addr":"[fd00:1122:3344:101::c]:60558","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":44,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:02.556720256Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:50709"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:02.557745493Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:02.589927269Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:02.640714249Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"2b50b53f-4ac6-4a21-9e93-dcf700687407","remote_addr":"[fd00:1122:3344:101::b]:50709","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":83929,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:02.641091495Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:48985"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:02.641110835Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"587280d0-f315-4cdc-8795-6e499e1576da","remote_addr":"[fd00:1122:3344:101::c]:60558","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":183,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:02.641154846Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:53313"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:02.642134922Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:02.642231904Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"013ec962-b223-486d-b757-b61509ed290d","remote_addr":"[fd00:1122:3344:101::b]:50709","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1236,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:02.642242394Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"d2604fef-3baf-4356-8885-4fe4018e50db","remote_addr":"[fd00:1122:3344:101::c]:53313","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":976,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:02.677615422Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:02.727801142Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"952f66ce-02e3-4186-95d8-32ebc6639375","remote_addr":"[fd00:1122:3344:101::a]:48985","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":86543,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:02.728142968Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"997bc9ca-1776-48c6-b36c-a65c57bf72f0","remote_addr":"[fd00:1122:3344:101::a]:48985","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":141,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:02.783292549Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:54735"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:02.783403211Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:02.821088227Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:02.873806598Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"beadc4b3-e780-4d9b-96a4-30b27d82bbcf","remote_addr":"[fd00:1122:3344:101::c]:54735","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":90447,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:02.874106073Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"53ced241-3c03-4536-b777-b2696c9176b4","remote_addr":"[fd00:1122:3344:101::c]:54735","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":120,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:17.52697651Z","hostname":"buskin","pid":1562,"uri":"/4b1d95e0-2253-4dc0-9406-d601ae11a340","method":"GET","req_id":"c3330bbe-f220-4871-b403-8263bf3c9dc7","remote_addr":"[fd00:1122:3344:101::d]:59584","local_addr":"[fd00:1122:3344:101::1]:46199","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":7060,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:28.741615939Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63675"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:28.741727771Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"874cf2ec-8ffb-4df8-984d-e0c6105ccbbb","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63675","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":58,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:32.529629943Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:41301"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:32.529829487Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"b6928cd7-c5e4-43e1-8114-9a2d4ef2ffde","remote_addr":"[fd00:1122:3344:101::b]:41301","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":174,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:32.566339223Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:34641"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:32.566405174Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"58db0f70-caf7-4d8f-a680-84f44bc6774d","remote_addr":"[fd00:1122:3344:101::b]:34641","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":43,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:32.573663953Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"ae2bed6a-5f8b-496c-b58d-e72d566009c3","remote_addr":"[fd00:1122:3344:101::b]:34641","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":78,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:32.596190681Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:58688"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:32.596243472Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"93d12d2f-f417-45fa-ac1c-0fd510538f12","remote_addr":"[fd00:1122:3344:101::a]:58688","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":18,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:32.60285371Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"9e883f25-ba24-44c8-8358-66c1f99f3f10","remote_addr":"[fd00:1122:3344:101::a]:58688","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":58,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:32.689906803Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:49748"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:32.689944663Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"bdb372a0-92c1-45a2-a5a3-93cddfedd95f","remote_addr":"[fd00:1122:3344:101::c]:49748","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":33,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:32.696299557Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"145de879-b5c8-4228-9526-cae9aca3e818","remote_addr":"[fd00:1122:3344:101::c]:49748","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":67,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:32.706117957Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:54844"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:32.70624393Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"374a177d-a3ff-491c-9220-556d34172f74","remote_addr":"[fd00:1122:3344:101::a]:54844","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":104,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:32.762265455Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:53032"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:32.762419718Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"392c2eb7-4a22-474d-bf94-14ffd80eafb5","remote_addr":"[fd00:1122:3344:101::c]:53032","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":129,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:32.78211007Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:33112"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:32.782207781Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:32.8182143Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:32.8739936Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"d7982c73-32c4-4d1a-9052-735ecb0780ee","remote_addr":"[fd00:1122:3344:101::b]:33112","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":91887,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:32.874271456Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:39736"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:32.874432598Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"b97ae14e-1ee0-420f-b1a6-239b4d14b740","remote_addr":"[fd00:1122:3344:101::b]:33112","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":159,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:32.874478769Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:32.902890773Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:49769"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:32.911345882Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:32.969255848Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:32.969319089Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"093d5ae3-9b31-40a3-95c0-e3dc02dc46de","remote_addr":"[fd00:1122:3344:101::c]:39736","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":94907,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:32.972948838Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"1990cb27-45eb-4449-a08c-95fdb6a14047","remote_addr":"[fd00:1122:3344:101::c]:39736","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":539,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:33.00301362Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:33.061398184Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"ceb28fc2-e0b8-4074-9254-4ff29e11aed7","remote_addr":"[fd00:1122:3344:101::a]:49769","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":158345,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:33.062281118Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"aebab6d8-0fed-4dd3-9e4c-c2715a2202ca","remote_addr":"[fd00:1122:3344:101::a]:49769","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":705,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:47.622766786Z","hostname":"buskin","pid":1562,"uri":"/4b1d95e0-2253-4dc0-9406-d601ae11a340","method":"GET","req_id":"b9c467de-eeeb-4848-b3f9-9ef5b1024bb2","remote_addr":"[fd00:1122:3344:101::d]:59584","local_addr":"[fd00:1122:3344:101::1]:46199","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":6460,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:58.776863307Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:58656"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:54:58.776950479Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"326d27d9-c513-4d21-9f4b-f5ee0d21a133","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:58656","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":49,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:02.339797093Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:44719"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:02.341081884Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"37c7364c-ae11-4097-a591-5d60b903b50a","remote_addr":"[fd00:1122:3344:101::b]:44719","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1246,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:02.34391151Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:57735"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:02.344060273Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"f4504ddb-a222-46e4-a3e1-941ff7700935","remote_addr":"[fd00:1122:3344:101::b]:57735","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":123,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:02.348048028Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"f9f9b6eb-3af8-4dac-82b4-7cb7ce641075","remote_addr":"[fd00:1122:3344:101::b]:44719","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":103,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:02.496188198Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:32871"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:02.49633952Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:02.53243111Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:02.590033681Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"09329889-28cd-4b1e-b087-3316c1d87cda","remote_addr":"[fd00:1122:3344:101::b]:32871","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":93786,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:02.590202704Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:64947"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:02.590477979Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"c11b8b1b-8428-4cb9-b2fd-18de6109003e","remote_addr":"[fd00:1122:3344:101::c]:64947","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":206,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:02.59053134Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"e25e849e-fd29-4404-930f-d6c72ab220ff","remote_addr":"[fd00:1122:3344:101::b]:32871","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":199,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:02.604243114Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:52296"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:02.604350975Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"152b35d4-5cb7-42ef-bec3-e363e68bbf3b","remote_addr":"[fd00:1122:3344:101::a]:52296","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":69,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:02.610642608Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:61364"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:02.610804971Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"8c89fdb6-165b-424c-9255-3519a7824742","remote_addr":"[fd00:1122:3344:101::a]:61364","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":111,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:02.61134793Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"6a6b920a-f743-4645-8373-a98f7bcc01d0","remote_addr":"[fd00:1122:3344:101::a]:52296","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":95,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:02.662818951Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:57340"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:02.662934883Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"42f0082a-081b-4126-baef-631f4c648c91","remote_addr":"[fd00:1122:3344:101::c]:57340","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":37,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:02.669217275Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"e3db2893-a4b2-430e-a42f-a2ba1545af84","remote_addr":"[fd00:1122:3344:101::c]:57340","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":575,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:02.745653965Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:64510"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:02.746368616Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:02.781844116Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:02.840069498Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"9b766255-c3d5-4bdb-a370-3443d618d20b","remote_addr":"[fd00:1122:3344:101::a]:64510","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":94357,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:02.84022372Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:62177"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:02.840400863Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:02.844033932Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"5a244111-9ae5-4f48-b0a7-efc28ae7712a","remote_addr":"[fd00:1122:3344:101::a]:64510","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":3624,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:02.878908522Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:02.935733261Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"8148b27e-c65a-4285-b5d0-dca8a3472293","remote_addr":"[fd00:1122:3344:101::c]:62177","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":95411,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:02.936126788Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"376812fe-548a-4ecb-ab9f-f09578e2208e","remote_addr":"[fd00:1122:3344:101::c]:62177","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":137,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:07.77776038Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:53467"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:07.777946763Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"07f0b330-4da2-4bb2-b8f5-7bee61224373","remote_addr":"[fd00:1122:3344:101::a]:53467","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":172,"response_code":"200"} {"msg":"Mapping virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:07.780433124Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:778","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 248, 131, 192])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(11432777) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:07.780456404Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"PUT","req_id":"f5fd5d4e-6057-4444-ba10-060010ed4cee","remote_addr":"[fd00:1122:3344:101::a]:53467","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":2350,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:07.908940084Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:45829"} {"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:07.918634243Z","hostname":"buskin","pid":1562,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:570","metadata":"InstanceMetadata { silo_id: 6011392c-84f5-4f85-a93c-1534826a6b77, project_id: aca8f70e-fc8c-4895-b26d-83364afed610 }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-08-27T19:55:07.498287Z }","instance_runtime":"InstanceRuntimeState { propolis_id: Some(ef97b3fe-ca3f-4936-ac46-f23927ef26c0 (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-08-27T19:55:07.542694449Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: da69e07c-f42f-415b-b4a8-07f9aa874b47, kind: Instance { id: 017162fa-e0d8-4736-b378-32380014261d }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 248, 131, 192])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(11432777), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: da69e07c-f42f-415b-b4a8-07f9aa874b47, kind: Instance { id: 017162fa-e0d8-4736-b378-32380014261d }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 248, 131, 192])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(11432777), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: da69e07c-f42f-415b-b4a8-07f9aa874b47, kind: Instance { id: 017162fa-e0d8-4736-b378-32380014261d }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 248, 131, 192])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(11432777), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(11432777))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: da69e07c-f42f-415b-b4a8-07f9aa874b47, kind: Instance { id: 017162fa-e0d8-4736-b378-32380014261d }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 248, 131, 192])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(11432777), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-6467185825\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 022d1f3d-9477-491c-aaca-6b37cf4a31cb, read_only_parent: Some(Volume { block_size: 512, id: 6d65d8e1-63e6-4e59-9cbd-56c213d4ded7, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 74b7946c-a5f0-4d0f-ba73-a97969169022, key: Some(\"2stdjhVLp29TTgZ76NF+TD72+qVM7ZOwP8HJq3kYiHE=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::17]:19001\", \"[fd00:1122:3344:101::1a]:19002\", \"[fd00:1122:3344:101::15]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 022d1f3d-9477-491c-aaca-6b37cf4a31cb, key: Some(\"3qk3xKK3nW1Q9GtKF66x0ZRnmAdMefASvPC+UZG7iUQ=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::1a]:19003\", \"[fd00:1122:3344:101::16]:19000\", \"[fd00:1122:3344:101::12]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAC6454bWRtZAADjnhtZAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAu+OeG1kbWQAA454bWQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiIwMTcxNjJmYS1lMGQ4LTQ3MzYtYjM3OC0zMjM4MDAxNDI2MWQiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJSDc0Y29uY3ZlSmhIaVZ1cjJsUTVkSlVsaWtxVU5FNnpkbE5UYVZ0WlBGWCJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }","propolis_id":"ef97b3fe-ca3f-4936-ac46-f23927ef26c0","instance_id":"017162fa-e0d8-4736-b378-32380014261d"} {"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:07.918726904Z","hostname":"buskin","pid":1562,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:603","propolis_id":"ef97b3fe-ca3f-4936-ac46-f23927ef26c0","instance_id":"017162fa-e0d8-4736-b378-32380014261d"} {"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:07.918859216Z","hostname":"buskin","pid":1562,"propolis_id":"ef97b3fe-ca3f-4936-ac46-f23927ef26c0","instance_id":"017162fa-e0d8-4736-b378-32380014261d","component":"InstanceManager","file":"sled-agent/src/instance.rs:990","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: da69e07c-f42f-415b-b4a8-07f9aa874b47, kind: Instance { id: 017162fa-e0d8-4736-b378-32380014261d }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 248, 131, 192])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(11432777), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: da69e07c-f42f-415b-b4a8-07f9aa874b47, kind: Instance { id: 017162fa-e0d8-4736-b378-32380014261d }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 248, 131, 192])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(11432777), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: da69e07c-f42f-415b-b4a8-07f9aa874b47, kind: Instance { id: 017162fa-e0d8-4736-b378-32380014261d }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 248, 131, 192])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(11432777), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(11432777))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: da69e07c-f42f-415b-b4a8-07f9aa874b47, kind: Instance { id: 017162fa-e0d8-4736-b378-32380014261d }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 248, 131, 192])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(11432777), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-6467185825\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 022d1f3d-9477-491c-aaca-6b37cf4a31cb, read_only_parent: Some(Volume { block_size: 512, id: 6d65d8e1-63e6-4e59-9cbd-56c213d4ded7, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 74b7946c-a5f0-4d0f-ba73-a97969169022, key: Some(\"2stdjhVLp29TTgZ76NF+TD72+qVM7ZOwP8HJq3kYiHE=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::17]:19001\", \"[fd00:1122:3344:101::1a]:19002\", \"[fd00:1122:3344:101::15]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 022d1f3d-9477-491c-aaca-6b37cf4a31cb, key: Some(\"3qk3xKK3nW1Q9GtKF66x0ZRnmAdMefASvPC+UZG7iUQ=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::1a]:19003\", \"[fd00:1122:3344:101::16]:19000\", \"[fd00:1122:3344:101::12]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAC6454bWRtZAADjnhtZAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAu+OeG1kbWQAA454bWQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiIwMTcxNjJmYS1lMGQ4LTQ3MzYtYjM3OC0zMjM4MDAxNDI2MWQiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJSDc0Y29uY3ZlSmhIaVZ1cjJsUTVkSlVsaWtxVU5FNnpkbE5UYVZ0WlBGWCJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-08-27T19:55:07.498287Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400, migration_id: None }","migration_id":"None","propolis_id":"ef97b3fe-ca3f-4936-ac46-f23927ef26c0","instance_id":"017162fa-e0d8-4736-b378-32380014261d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:07.918986168Z","hostname":"buskin","pid":1562,"uri":"/vmms/ef97b3fe-ca3f-4936-ac46-f23927ef26c0","method":"PUT","req_id":"f564edf6-9bce-405d-b122-a334f173908e","remote_addr":"[fd00:1122:3344:101::a]:45829","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":9965,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:07.993417915Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:60890"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:07.995348556Z","hostname":"buskin","pid":1562,"uri":"/vmms/ef97b3fe-ca3f-4936-ac46-f23927ef26c0/state","method":"PUT","req_id":"c0a942cc-5191-423d-a39f-a0260c8f7560","remote_addr":"[fd00:1122:3344:101::a]:60890","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1873,"response_code":"200"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:07.999666407Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:424","port":"Port { inner: PortInner(PortData { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 248, 131, 192]), slot: 0, vni: Vni { inner: 11432777 }, subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 } }) }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:07.999712398Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Configuring new Omicron zone: oxz_propolis-server_ef97b3fe-ca3f-4936-ac46-f23927ef26c0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:08.037272421Z","hostname":"buskin","pid":1562,"instance_id":"017162fa-e0d8-4736-b378-32380014261d","propolis_id":"ef97b3fe-ca3f-4936-ac46-f23927ef26c0","instance_id":"017162fa-e0d8-4736-b378-32380014261d","component":"InstanceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_propolis-server_ef97b3fe-ca3f-4936-ac46-f23927ef26c0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:08.057676005Z","hostname":"buskin","pid":1562,"instance_id":"017162fa-e0d8-4736-b378-32380014261d","propolis_id":"ef97b3fe-ca3f-4936-ac46-f23927ef26c0","instance_id":"017162fa-e0d8-4736-b378-32380014261d","component":"InstanceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_propolis-server_ef97b3fe-ca3f-4936-ac46-f23927ef26c0:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:09.431884597Z","hostname":"buskin","pid":1562,"instance_id":"017162fa-e0d8-4736-b378-32380014261d","propolis_id":"ef97b3fe-ca3f-4936-ac46-f23927ef26c0","instance_id":"017162fa-e0d8-4736-b378-32380014261d","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_propolis-server_ef97b3fe-ca3f-4936-ac46-f23927ef26c0 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:09.432064337Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_ef97b3fe-ca3f-4936-ac46-f23927ef26c0","instance_id":"017162fa-e0d8-4736-b378-32380014261d","propolis_id":"ef97b3fe-ca3f-4936-ac46-f23927ef26c0","instance_id":"017162fa-e0d8-4736-b378-32380014261d","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_propolis-server_ef97b3fe-ca3f-4936-ac46-f23927ef26c0\") failed: Property not found. retry in 65.676647ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:55:09.657089462Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_ef97b3fe-ca3f-4936-ac46-f23927ef26c0","instance_id":"017162fa-e0d8-4736-b378-32380014261d","propolis_id":"ef97b3fe-ca3f-4936-ac46-f23927ef26c0","instance_id":"017162fa-e0d8-4736-b378-32380014261d","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_ef97b3fe-ca3f-4936-ac46-f23927ef26c0\") failed: Property not found. retry in 127.834795ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:55:09.752260647Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_ef97b3fe-ca3f-4936-ac46-f23927ef26c0","instance_id":"017162fa-e0d8-4736-b378-32380014261d","propolis_id":"ef97b3fe-ca3f-4936-ac46-f23927ef26c0","instance_id":"017162fa-e0d8-4736-b378-32380014261d","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_ef97b3fe-ca3f-4936-ac46-f23927ef26c0\") failed: Property not found. retry in 260.176971ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:55:09.901046209Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_ef97b3fe-ca3f-4936-ac46-f23927ef26c0","instance_id":"017162fa-e0d8-4736-b378-32380014261d","propolis_id":"ef97b3fe-ca3f-4936-ac46-f23927ef26c0","instance_id":"017162fa-e0d8-4736-b378-32380014261d","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_ef97b3fe-ca3f-4936-ac46-f23927ef26c0\") failed: Property not found. retry in 499.129275ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:55:10.17586784Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_ef97b3fe-ca3f-4936-ac46-f23927ef26c0","instance_id":"017162fa-e0d8-4736-b378-32380014261d","propolis_id":"ef97b3fe-ca3f-4936-ac46-f23927ef26c0","instance_id":"017162fa-e0d8-4736-b378-32380014261d","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_ef97b3fe-ca3f-4936-ac46-f23927ef26c0\") failed: Property not found. retry in 686.255714ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:55:10.684586303Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_ef97b3fe-ca3f-4936-ac46-f23927ef26c0","instance_id":"017162fa-e0d8-4736-b378-32380014261d","propolis_id":"ef97b3fe-ca3f-4936-ac46-f23927ef26c0","instance_id":"017162fa-e0d8-4736-b378-32380014261d","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_ef97b3fe-ca3f-4936-ac46-f23927ef26c0\") failed: Property not found. retry in 733.758808ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:55:11.382086422Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_ef97b3fe-ca3f-4936-ac46-f23927ef26c0","instance_id":"017162fa-e0d8-4736-b378-32380014261d","propolis_id":"ef97b3fe-ca3f-4936-ac46-f23927ef26c0","instance_id":"017162fa-e0d8-4736-b378-32380014261d","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_ef97b3fe-ca3f-4936-ac46-f23927ef26c0\") failed: Property not found. retry in 654.699521ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:55:12.127765098Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_ef97b3fe-ca3f-4936-ac46-f23927ef26c0","instance_id":"017162fa-e0d8-4736-b378-32380014261d","propolis_id":"ef97b3fe-ca3f-4936-ac46-f23927ef26c0","instance_id":"017162fa-e0d8-4736-b378-32380014261d","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_ef97b3fe-ca3f-4936-ac46-f23927ef26c0\") failed: Property not found. retry in 788.801219ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:55:12.793977895Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_ef97b3fe-ca3f-4936-ac46-f23927ef26c0","instance_id":"017162fa-e0d8-4736-b378-32380014261d","propolis_id":"ef97b3fe-ca3f-4936-ac46-f23927ef26c0","instance_id":"017162fa-e0d8-4736-b378-32380014261d","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_ef97b3fe-ca3f-4936-ac46-f23927ef26c0\") failed: Property not found. retry in 1.237655425s","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:55:13.594326805Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_ef97b3fe-ca3f-4936-ac46-f23927ef26c0","instance_id":"017162fa-e0d8-4736-b378-32380014261d","propolis_id":"ef97b3fe-ca3f-4936-ac46-f23927ef26c0","instance_id":"017162fa-e0d8-4736-b378-32380014261d","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Started propolis in zone: oxz_propolis-server_ef97b3fe-ca3f-4936-ac46-f23927ef26c0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:14.859055763Z","hostname":"buskin","pid":1562,"instance_id":"017162fa-e0d8-4736-b378-32380014261d","propolis_id":"ef97b3fe-ca3f-4936-ac46-f23927ef26c0","instance_id":"017162fa-e0d8-4736-b378-32380014261d","component":"InstanceManager","file":"sled-agent/src/instance.rs:1448"} {"msg":"wait for service Some(\"oxz_propolis-server_ef97b3fe-ca3f-4936-ac46-f23927ef26c0\") failed: Property not found. retry in 29.516697ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:55:14.867964839Z","hostname":"buskin","pid":1562,"instance_id":"017162fa-e0d8-4736-b378-32380014261d","propolis_id":"ef97b3fe-ca3f-4936-ac46-f23927ef26c0","instance_id":"017162fa-e0d8-4736-b378-32380014261d","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_ef97b3fe-ca3f-4936-ac46-f23927ef26c0\") failed: Property not found. retry in 58.111566ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:55:14.906920305Z","hostname":"buskin","pid":1562,"instance_id":"017162fa-e0d8-4736-b378-32380014261d","propolis_id":"ef97b3fe-ca3f-4936-ac46-f23927ef26c0","instance_id":"017162fa-e0d8-4736-b378-32380014261d","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_ef97b3fe-ca3f-4936-ac46-f23927ef26c0\") failed: Property not found. retry in 283.557699ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-27T19:55:14.975003208Z","hostname":"buskin","pid":1562,"instance_id":"017162fa-e0d8-4736-b378-32380014261d","propolis_id":"ef97b3fe-ca3f-4936-ac46-f23927ef26c0","instance_id":"017162fa-e0d8-4736-b378-32380014261d","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:15.268980112Z","hostname":"buskin","pid":1562,"instance_id":"017162fa-e0d8-4736-b378-32380014261d","propolis_id":"ef97b3fe-ca3f-4936-ac46-f23927ef26c0","instance_id":"017162fa-e0d8-4736-b378-32380014261d","component":"InstanceManager","file":"sled-agent/src/instance.rs:1457"} {"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:15.308294075Z","hostname":"buskin","pid":1562,"instance_id":"017162fa-e0d8-4736-b378-32380014261d","propolis_id":"ef97b3fe-ca3f-4936-ac46-f23927ef26c0","instance_id":"017162fa-e0d8-4736-b378-32380014261d","component":"InstanceManager","file":"sled-agent/src/instance.rs:1481"} {"msg":"Sending ensure request to propolis: InstanceEnsureRequest { cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAC6454bWRtZAADjnhtZAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAu+OeG1kbWQAA454bWQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiIwMTcxNjJmYS1lMGQ4LTQ3MzYtYjM3OC0zMjM4MDAxNDI2MWQiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJSDc0Y29uY3ZlSmhIaVZ1cjJsUTVkSlVsaWtxVU5FNnpkbE5UYVZ0WlBGWCJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"), disks: [DiskRequest { device: \"nvme\", name: \"disk-6467185825\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 022d1f3d-9477-491c-aaca-6b37cf4a31cb, read_only_parent: Some(Volume { block_size: 512, id: 6d65d8e1-63e6-4e59-9cbd-56c213d4ded7, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 74b7946c-a5f0-4d0f-ba73-a97969169022, key: Some(\"2stdjhVLp29TTgZ76NF+TD72+qVM7ZOwP8HJq3kYiHE=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::17]:19001\", \"[fd00:1122:3344:101::1a]:19002\", \"[fd00:1122:3344:101::15]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 022d1f3d-9477-491c-aaca-6b37cf4a31cb, key: Some(\"3qk3xKK3nW1Q9GtKF66x0ZRnmAdMefASvPC+UZG7iUQ=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::1a]:19003\", \"[fd00:1122:3344:101::16]:19000\", \"[fd00:1122:3344:101::12]:19000\"] } }] } }], migrate: None, nics: [NetworkInterfaceRequest { name: \"opte6\", slot: Slot(0) }], properties: InstanceProperties { bootrom_id: 00000000-0000-0000-0000-000000000000, description: \"Test description\", id: 017162fa-e0d8-4736-b378-32380014261d, image_id: 00000000-0000-0000-0000-000000000000, memory: 1024, metadata: InstanceMetadata { project_id: aca8f70e-fc8c-4895-b26d-83364afed610, silo_id: 6011392c-84f5-4f85-a93c-1534826a6b77, sled_id: 4b1d95e0-2253-4dc0-9406-d601ae11a340, sled_model: \"i86pc\", sled_revision: 0, sled_serial: \"buskin\" }, name: \"localshark\", vcpus: 2 } }","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:15.308550319Z","hostname":"buskin","pid":1562,"instance_id":"017162fa-e0d8-4736-b378-32380014261d","propolis_id":"ef97b3fe-ca3f-4936-ac46-f23927ef26c0","instance_id":"017162fa-e0d8-4736-b378-32380014261d","component":"InstanceManager","file":"sled-agent/src/instance.rs:724"} {"msg":"result of instance_ensure call is Err(Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d79ab988-e7c4-4413-9380-fb660ccb5ac0\", \"content-length\": \"124\", \"date\": \"Tue, 27 Aug 2024 19:55:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d79ab988-e7c4-4413-9380-fb660ccb5ac0\" })","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:15.377332003Z","hostname":"buskin","pid":1562,"instance_id":"017162fa-e0d8-4736-b378-32380014261d","propolis_id":"ef97b3fe-ca3f-4936-ac46-f23927ef26c0","instance_id":"017162fa-e0d8-4736-b378-32380014261d","component":"InstanceManager","file":"sled-agent/src/instance.rs:726"} {"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:15.377376354Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:904","id":"RouterId { vni: Vni(11432777), kind: System }"} {"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:15.377388104Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:904","id":"RouterId { vni: Vni(11432777), kind: Custom(V4(Ipv4Net { addr: 172.30.0.0, width: 22 })) }"} WARNING: Failed to delete the xde device. It must be deleted out of band, and it will not be possible to recreate the xde device until then. Error: CommandError(DeleteXde, System { errno: 16, msg: "failed to destroy DLS devnet: 16" }) {"msg":"vmm setup failed: Err(Propolis(Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d79ab988-e7c4-4413-9380-fb660ccb5ac0\", \"content-length\": \"124\", \"date\": \"Tue, 27 Aug 2024 19:55:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d79ab988-e7c4-4413-9380-fb660ccb5ac0\" }))","v":0,"name":"SledAgent","level":50,"time":"2024-08-27T19:55:15.387369607Z","hostname":"buskin","pid":1562,"instance_id":"017162fa-e0d8-4736-b378-32380014261d","propolis_id":"ef97b3fe-ca3f-4936-ac46-f23927ef26c0","instance_id":"017162fa-e0d8-4736-b378-32380014261d","component":"InstanceManager","file":"sled-agent/src/instance.rs:1288"} {"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:17.151022579Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_ef97b3fe-ca3f-4936-ac46-f23927ef26c0","instance_id":"017162fa-e0d8-4736-b378-32380014261d","propolis_id":"ef97b3fe-ca3f-4936-ac46-f23927ef26c0","instance_id":"017162fa-e0d8-4736-b378-32380014261d","component":"InstanceManager","file":"illumos-utils/src/zone.rs:270"} {"msg":"Stopped and uninstalled zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:17.15107556Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_ef97b3fe-ca3f-4936-ac46-f23927ef26c0","instance_id":"017162fa-e0d8-4736-b378-32380014261d","propolis_id":"ef97b3fe-ca3f-4936-ac46-f23927ef26c0","instance_id":"017162fa-e0d8-4736-b378-32380014261d","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:865"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:17.624220232Z","hostname":"buskin","pid":1562,"uri":"/4b1d95e0-2253-4dc0-9406-d601ae11a340","method":"GET","req_id":"981bb1a5-c312-4489-b8ba-fe4f36b2166f","remote_addr":"[fd00:1122:3344:101::d]:59584","local_addr":"[fd00:1122:3344:101::1]:46199","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":7810,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-08-27T19:55:25.309096699Z","hostname":"buskin","pid":1562,"component":"kstat-sampler-worker","sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:490","error":"NoSuchKstat","id":"12973599944580669215"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:28.80934384Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36354"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:28.809444361Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"f37b0b78-48bc-42b0-ba5f-6e3596ea1b3f","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36354","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":52,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:32.584185108Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:54086"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:32.584448352Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"1b7f2f22-8abb-47c0-8a57-f344972853ae","remote_addr":"[fd00:1122:3344:101::a]:54086","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":233,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:32.590821027Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:47282"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:32.590949039Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"90782fae-f897-49bf-bd0b-7b17fcd3c765","remote_addr":"[fd00:1122:3344:101::a]:47282","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":60,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:32.597581797Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"6f4d0bda-a05c-4012-9389-b37e764c32f7","remote_addr":"[fd00:1122:3344:101::a]:47282","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":114,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:32.60325648Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:38470"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:32.604811275Z","hostname":"buskin","pid":1562,"uri":"/vmms/ef97b3fe-ca3f-4936-ac46-f23927ef26c0/state","method":"GET","req_id":"daaad549-8222-4627-8cdb-279b86f09764","remote_addr":"[fd00:1122:3344:101::a]:38470","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1530,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:32.627971754Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:40337"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:32.628140477Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"3980e2b3-7b9d-48b5-b068-b221a0be6d43","remote_addr":"[fd00:1122:3344:101::c]:40337","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":88,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:32.634187385Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"48167bea-6c5a-4a4c-9c93-a9ce935693d1","remote_addr":"[fd00:1122:3344:101::c]:40337","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":128,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:32.640068141Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:60550"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:32.640248224Z","hostname":"buskin","pid":1562,"uri":"/vmms/ef97b3fe-ca3f-4936-ac46-f23927ef26c0/state","method":"GET","req_id":"3858dfb0-9d8e-4c2b-ad79-765e95f5adb8","remote_addr":"[fd00:1122:3344:101::b]:60550","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":100,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:32.644326681Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:64395"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:32.644477084Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"e28fc34c-a0c4-4080-9020-460707cf4576","remote_addr":"[fd00:1122:3344:101::c]:64395","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":109,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:32.689908246Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:50277"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:32.689983807Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"61e039b9-8b0f-4462-a7dc-130e5b5a4181","remote_addr":"[fd00:1122:3344:101::b]:50277","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":41,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:32.697793695Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"e1e4b609-bab9-4b68-9339-01177694b85d","remote_addr":"[fd00:1122:3344:101::b]:50277","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":87,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:32.715672497Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:45875"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:32.715806259Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"4f476433-2aa2-446e-ad7f-4eaea37d265d","remote_addr":"[fd00:1122:3344:101::b]:45875","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":103,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:32.803597494Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:51620"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:32.803713156Z","hostname":"buskin","pid":1562,"uri":"/vmms/ef97b3fe-ca3f-4936-ac46-f23927ef26c0/state","method":"GET","req_id":"d04e48d6-6209-4027-849e-fdb794ea5c0c","remote_addr":"[fd00:1122:3344:101::c]:51620","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":85,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:32.895173791Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:52923"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:32.895284212Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:32.931112918Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:32.98385802Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"ec2b4cc0-9892-46e1-b8b6-eaac256abf6d","remote_addr":"[fd00:1122:3344:101::b]:52923","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":88625,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:32.984072363Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:41566"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:32.984161865Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:44767"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:32.984299797Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:32.988004748Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"2cb4fa48-7abd-4a7e-bdc5-2cd484616d71","remote_addr":"[fd00:1122:3344:101::b]:52923","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":3789,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:33.022783926Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:33.062943763Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:56533"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:33.063273388Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"0e3ce074-4c99-40c6-8631-d5cb04034ef5","remote_addr":"[fd00:1122:3344:101::a]:56533","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":236,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:33.070878592Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"00d1d58b-3189-4950-9c66-506691b1b60a","remote_addr":"[fd00:1122:3344:101::a]:56533","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":795,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:33.078324824Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:58520"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:33.078583468Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"3f83dbeb-e459-433a-b76f-02bf5d577c00","remote_addr":"[fd00:1122:3344:101::a]:58520","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":210,"response_code":"200"} {"msg":"Clearing mapping of virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:33.079023845Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:818","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 248, 131, 192])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(11432777) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:33.079049886Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"DELETE","req_id":"90e7c74c-2eca-4969-b8ce-c02312c3b6e5","remote_addr":"[fd00:1122:3344:101::a]:58520","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":245,"response_code":"204"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:33.08356676Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:33.08358498Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"d7424247-9c55-4cfb-995b-5484807bd607","remote_addr":"[fd00:1122:3344:101::a]:41566","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":99409,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:33.088669443Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"98d69b9f-81e8-4cc3-81b3-dcf368813460","remote_addr":"[fd00:1122:3344:101::a]:41566","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":708,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:33.121274286Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:33.176419917Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"31f04acb-4287-4598-989b-2ecd30228a02","remote_addr":"[fd00:1122:3344:101::c]:44767","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":192106,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:33.176815093Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"8c458859-eb69-484b-86b0-9e0b61c29715","remote_addr":"[fd00:1122:3344:101::c]:44767","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":145,"response_code":"200"} {"msg":"running automatic periodic zone bundle cleanup","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:33.60250549Z","hostname":"buskin","pid":1562,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:197"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:33.602541111Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-08-27T19:55:33.64412487Z","hostname":"buskin","pid":1562,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:186","time_to_next_cleanup":"599.999995649s","next_cleanup":"Instant { tv_sec: 1772, tv_nsec: 934324809 }"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-08-27T19:55:35.311874245Z","hostname":"buskin","pid":1562,"component":"kstat-sampler-worker","sled_id":"4b1d95e0-2253-4dc0-9406-d601ae11a340","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:490","error":"NoSuchKstat","id":"12973599944580669215"}