[ Aug 23 21:51:57 Enabled. ] [ Aug 23 21:51:57 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Aug 23 21:51:57 Method "start" exited with status 0. ] [ Aug 23 21:51:57 Rereading configuration. ] [ Aug 23 21:51:57 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:57.591234932Z","hostname":"buskin","pid":1562,"file":"illumos-utils/src/running_zone.rs:113"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:57.591379035Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:57.646264964Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:57.646311345Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:57.759650435Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:57.768425823Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:57.776567291Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:58.379965859Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fdb0:18c0:4d0d:9fb2::, width: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:58.518113109Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:126"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:58.51816973Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:138"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:58.518215321Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:151"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:58.518330203Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:173","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:58.518347393Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:520"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:58.518357813Z","hostname":"buskin","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:716"} {"msg":"Generated i86pc baseboard Pc { identifier: \"buskin\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:58.521340724Z","hostname":"buskin","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:623"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:51:58.521348354Z","hostname":"buskin","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:765"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:58.521357724Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:187"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:58.521361394Z","hostname":"buskin","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:824"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:58.521375704Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:58.521410135Z","hostname":"buskin","pid":1562,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:58.521483806Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:58.521505356Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:58.521519897Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:58.531950083Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:58.531983824Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:371","id":"156808b0-3956-455e-a8e1-2217e0afca54 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:58.632692129Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:307","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 156808b0-3956-455e-a8e1-2217e0afca54 (zpool), kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:58.840708542Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:58.840749073Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:58.840757953Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:58.840768963Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:58.840777063Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:58.840797574Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:58.840802824Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:58.849629733Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:58.849654264Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:371","id":"fe7217f3-1112-4ad5-a4fc-b58965332aab (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:58.938233494Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:307","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: fe7217f3-1112-4ad5-a4fc-b58965332aab (zpool), kind: Internal }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.143955127Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_0.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.143995218Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 156808b0-3956-455e-a8e1-2217e0afca54 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: fe7217f3-1112-4ad5-a4fc-b58965332aab (zpool), kind: Internal } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144030169Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144038119Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144044289Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144049339Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.14407802Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_1.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.14408717Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 156808b0-3956-455e-a8e1-2217e0afca54 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: fe7217f3-1112-4ad5-a4fc-b58965332aab (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.14410072Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.14410494Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.14410883Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.14412379Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144128321Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_2.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144163421Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 156808b0-3956-455e-a8e1-2217e0afca54 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: fe7217f3-1112-4ad5-a4fc-b58965332aab (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144182591Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144186722Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144190742Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144195722Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144201012Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_3.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144214402Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144219812Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 156808b0-3956-455e-a8e1-2217e0afca54 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: fe7217f3-1112-4ad5-a4fc-b58965332aab (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144245752Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144250492Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144254663Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144259393Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_4.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144262793Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 156808b0-3956-455e-a8e1-2217e0afca54 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: fe7217f3-1112-4ad5-a4fc-b58965332aab (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144279303Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144283583Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144288923Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144293353Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144298393Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_5.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144301713Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 156808b0-3956-455e-a8e1-2217e0afca54 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: fe7217f3-1112-4ad5-a4fc-b58965332aab (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144327164Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144331364Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144335194Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144341884Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144347454Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_6.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144351144Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 156808b0-3956-455e-a8e1-2217e0afca54 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: fe7217f3-1112-4ad5-a4fc-b58965332aab (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144372094Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144378115Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144382245Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144388255Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144391835Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_7.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144395825Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144401245Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 156808b0-3956-455e-a8e1-2217e0afca54 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: fe7217f3-1112-4ad5-a4fc-b58965332aab (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144424306Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144439706Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144444266Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144448076Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_8.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144453216Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144457066Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144461236Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 156808b0-3956-455e-a8e1-2217e0afca54 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: fe7217f3-1112-4ad5-a4fc-b58965332aab (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144483576Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144487956Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144493067Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:97"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144504497Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:99"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144510497Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 156808b0-3956-455e-a8e1-2217e0afca54 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: fe7217f3-1112-4ad5-a4fc-b58965332aab (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144536537Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144540917Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144544228Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144547588Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:211"} {"msg":"No ledger in /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144551077Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144571438Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144578978Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144588148Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144595518Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144603029Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144624239Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:216"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.144628669Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:231"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.155087346Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:731"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.155115227Z","hostname":"buskin","pid":1562,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:186","time_to_next_cleanup":"599.999931799s","next_cleanup":"Instant { tv_sec: 2070, tv_nsec: 161959806 }"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.155165337Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3861"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.155195368Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"No ledger in /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.155254589Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.155270349Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.158227039Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:205"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.15829509Z","hostname":"buskin","pid":1562,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:44","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.1.164,\n last: 10.151.1.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.1.164,\n 10.151.1.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.1.100,\n vlan_id: None,\n local_pref: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n lldp: None,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.158503094Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1053"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.158510354Z","hostname":"buskin","pid":1562,"component":"DnsResolver","component":"RSS","file":"internal-dns/src/resolver.rs:73","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.158610656Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"No ledger in /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.158653056Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.158663236Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.158668037Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1113"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.158691117Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"No ledger in /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.158701057Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.158714097Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.158718348Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1154"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.158722048Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:155"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:9fb2::1","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.158728158Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:128"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.158743918Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Writing ledger to /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.158769678Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.159076063Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Sled plan written to storage: Plan {\n rack_id: da7dff3e-1935-4d21-ab58-793c0302e3ea,\n sleds: {\n [fdb0:18c0:4d0d:9fb2::1]:12346: StartSledAgentRequest {\n generation: 0,\n schema_version: 1,\n body: StartSledAgentRequestBody {\n id: 5050fe14-0845-40b2-872d-d70a5b2b56be,\n rack_id: da7dff3e-1935-4d21-ab58-793c0302e3ea,\n use_trust_quorum: false,\n is_lrtq_learner: false,\n subnet: Ipv6Subnet {\n net: Ipv6Net {\n addr: fd00:1122:3344:101::,\n width: 64,\n },\n },\n },\n },\n },\n config: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.1.164,\n last: 10.151.1.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.1.164,\n 10.151.1.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.1.100,\n vlan_id: None,\n local_pref: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n lldp: None,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.159379779Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:175"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.159397299Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1187"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.15944189Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.15945693Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.159641663Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.159771615Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.159804156Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:217","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 5050fe14-0845-40b2-872d-d70a5b2b56be, rack_id: da7dff3e-1935-4d21-ab58-793c0302e3ea, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.159893397Z","hostname":"buskin","pid":1562,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:58752","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:66"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 5050fe14-0845-40b2-872d-d70a5b2b56be, rack_id: da7dff3e-1935-4d21-ab58-793c0302e3ea, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.1600523Z","hostname":"buskin","pid":1562,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:359"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.16006316Z","hostname":"buskin","pid":1562,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:374"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.160374616Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/server.rs:44"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.160416246Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:101::, width: 64 }"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.160427526Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.160431677Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:574"} {"msg":"No ledger in /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.160443657Z","hostname":"buskin","pid":1562,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.160457687Z","hostname":"buskin","pid":1562,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.160462367Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:563"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.160465957Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:591"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.160469737Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:395"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.160479537Z","hostname":"buskin","pid":1562,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:73","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.160495998Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.160505048Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:481","result":"DisksManagementResult { status: [] }"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.162541352Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"Starting switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.162607453Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4540"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.192368447Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.21141015Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.22914648Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.242272763Z","hostname":"buskin","pid":1562,"sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:388"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.242316513Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.242339544Z","hostname":"buskin","pid":1562,"sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:400"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.242356814Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 138.22301ms)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.24566837Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"62604daf-1d0a-47b6-9cc2-7f7141a57498\", \"content-length\": \"124\", \"date\": \"Fri, 23 Aug 2024 21:51:58 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"62604daf-1d0a-47b6-9cc2-7f7141a57498\" }"} {"msg":"attempting to create encrypted zvol: name=\"oxi_156808b0-3956-455e-a8e1-2217e0afca54/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.256761998Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"successfully created encrypted zvol: name=\"oxi_156808b0-3956-455e-a8e1-2217e0afca54/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.280077653Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_156808b0-3956-455e-a8e1-2217e0afca54/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.280119233Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.285469154Z","hostname":"buskin","pid":1562,"sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:415"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.285477764Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:126"} {"msg":"Ensuring dataset oxi_156808b0-3956-455e-a8e1-2217e0afca54/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.285486594Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:136"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.362702432Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:170"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.384813617Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 175.85711ms)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.385407247Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"71fd074a-c6e3-4ee0-9440-5532952e8966\", \"content-length\": \"124\", \"date\": \"Fri, 23 Aug 2024 21:51:59 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"71fd074a-c6e3-4ee0-9440-5532952e8966\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.56218255Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 198.267209ms)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.562698059Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"085723c6-fe6a-4818-80a7-585ff16afcd3\", \"content-length\": \"124\", \"date\": \"Fri, 23 Aug 2024 21:51:59 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"085723c6-fe6a-4818-80a7-585ff16afcd3\" }"} {"msg":"Mounting oxi_156808b0-3956-455e-a8e1-2217e0afca54/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.582361342Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:178"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.596017953Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.596085255Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.596129795Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.596145285Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:192"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.762288999Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 239.275009ms)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:51:59.763041912Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bac20cc6-bd5f-44d7-af52-2539d82ceb61\", \"content-length\": \"124\", \"date\": \"Fri, 23 Aug 2024 21:51:59 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"bac20cc6-bd5f-44d7-af52-2539d82ceb61\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:00.002820752Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 185.204786ms)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:00.003542935Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f585cf53-5379-4f25-b2ef-72adc4c4870e\", \"content-length\": \"124\", \"date\": \"Fri, 23 Aug 2024 21:51:59 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f585cf53-5379-4f25-b2ef-72adc4c4870e\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:00.190626673Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 135.48865ms)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:00.191295054Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"da6f1b6e-4b9e-4149-973b-57f7a8b759ec\", \"content-length\": \"124\", \"date\": \"Fri, 23 Aug 2024 21:51:59 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"da6f1b6e-4b9e-4149-973b-57f7a8b759ec\" }"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:00.323563454Z","hostname":"buskin","pid":1562,"sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:90"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:00.328835673Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:00.329520525Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:46625","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:205","component":"producer-server","sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:205"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:00.329548945Z","hostname":"buskin","pid":1562,"component":"internal-dns-resolver","file":"internal-dns/src/resolver.rs:73","component":"producer-server","sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"internal-dns/src/resolver.rs:73","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:00.329623577Z","hostname":"buskin","pid":1562,"file":"oximeter/producer/src/lib.rs:283","component":"producer-server","sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"oximeter/producer/src/lib.rs:283","interval":"30s","address":"[fd00:1122:3344:101::1]:46625","producer_id":"5050fe14-0845-40b2-872d-d70a5b2b56be"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:00.330723765Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"203.773123ms"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:00.331891805Z","hostname":"buskin","pid":1562,"component":"VmmReservoirManager","sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:00.331906916Z","hostname":"buskin","pid":1562,"component":"VmmReservoirManager","sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:00.331972127Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:915","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:00.331981257Z","hostname":"buskin","pid":1562,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/src/resolver.rs:73","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:00.536593182Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"263.523668ms"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:00.802520195Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"1.03732647s"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:01.842717431Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"1.977895864s"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:02.681220401Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2634"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:02.681246312Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2852"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:02.681251902Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2504"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:02.681256592Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2551"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:02.681260772Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2933"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:02.681264892Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2975"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:02.683145804Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2777"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:02.684402635Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2545"} {"msg":"Profile for oxz_switch:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:02.684508127Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:02.684750921Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 53.449223ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:03.062299035Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 84.898955ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:03.239832061Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 212.538565ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:03.430100103Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 592.577042ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:03.666759561Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:03.821756556Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"3.88271509s"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.023242138s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:04.394284102Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 703.928318ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:05.48837785Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 682.468545ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:06.381903242Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.243357093s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:07.280649152Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:07.706101827Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"6.636376331s"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:09.009203535Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:205"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:09.009252096Z","hostname":"buskin","pid":1562,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:429"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:09.009265076Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Writing ledger to /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:09.009285687Z","hostname":"buskin","pid":1562,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:09.00949913Z","hostname":"buskin","pid":1562,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:09.009685303Z","hostname":"buskin","pid":1562,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:09.009718634Z","hostname":"buskin","pid":1562,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV2 { rack_subnet: Ipv6Net { addr: fd00:1122:3344:100::, width: 56 }, infra_ip_first: 10.151.1.174, infra_ip_last: 10.151.1.174, ports: [PortConfigV2 { routes: [RouteConfig { destination: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), nexthop: 10.151.1.100, vlan_id: None, local_pref: None }], addresses: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false, lldp: None }], bgp: [], bfd: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:09.009745404Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3861"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:09.009750975Z","hostname":"buskin","pid":1562,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:58752","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:73"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:09.009829526Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:108","peer":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:09.009836006Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:09.24215542Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"No ledger in /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:09.242238202Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:09.242276822Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:09.242316853Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:09.242332313Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:09.247623293Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4182","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:09.247642743Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4198"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:09.247650504Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:534"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:09.500202301Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:46501"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:09.500508006Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:09.500668248Z","hostname":"buskin","pid":1562,"uri":"/inventory","method":"GET","req_id":"26071094-3b45-4b8d-833e-9e89ba5bd968","remote_addr":"[fd00:1122:3344:101::1]:46501","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":323,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:09.531078043Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:37130"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:09.531196445Z","hostname":"buskin","pid":1562,"uri":"/sled-role","method":"GET","req_id":"a4728655-1bed-4078-9133-1612fadf1d57","remote_addr":"[fd00:1122:3344:101::1]:37130","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":57,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:09.531902887Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Writing ledger to /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:09.531920438Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:09.532218003Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:09.532432656Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:884"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 11713d4c-ac38-409f-b44b-50d6a207b503, pool_id: c4828395-89b1-4653-93a7-55b242871c48 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: cce62975-3bcc-483e-bd28-228e5a44011a, pool_id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: 940fedf0-b769-445c-983e-f4f5c9010dce, pool_id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 4ee4b373-3198-4629-b431-9a7495b69ca1, pool_id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: ac5eacfe-7b2a-4ada-b65b-050c9807cd1e, pool_id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: dd1eb680-3131-4e02-869e-de5fafaf42f9, pool_id: dad70eae-b83f-4c3a-a32e-2a6a8946803c (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 7dd96546-1f9b-41d8-bb80-b12a0f1de1a5, pool_id: ec50006e-a21c-4820-b439-48f6d6b60c97 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 8e3c003d-8926-48ad-98e1-438bed02e673, pool_id: e1a016b1-16ad-48d4-a26f-2e916f51bc80 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: bc3aa74f-31f1-4cb7-a638-01dc0627d7f2, pool_id: 7ac03450-9a27-4ec4-9669-9eb798da4419 (zpool) }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:09.56510168Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:357"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:09.565229992Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:59727"} {"msg":"physical disks ensure","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:09.565479286Z","hostname":"buskin","pid":1562,"sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:833"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 11713d4c-ac38-409f-b44b-50d6a207b503, pool_id: c4828395-89b1-4653-93a7-55b242871c48 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: cce62975-3bcc-483e-bd28-228e5a44011a, pool_id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: 940fedf0-b769-445c-983e-f4f5c9010dce, pool_id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 4ee4b373-3198-4629-b431-9a7495b69ca1, pool_id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: ac5eacfe-7b2a-4ada-b65b-050c9807cd1e, pool_id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: dd1eb680-3131-4e02-869e-de5fafaf42f9, pool_id: dad70eae-b83f-4c3a-a32e-2a6a8946803c (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 7dd96546-1f9b-41d8-bb80-b12a0f1de1a5, pool_id: ec50006e-a21c-4820-b439-48f6d6b60c97 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 8e3c003d-8926-48ad-98e1-438bed02e673, pool_id: e1a016b1-16ad-48d4-a26f-2e916f51bc80 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: bc3aa74f-31f1-4cb7-a638-01dc0627d7f2, pool_id: 7ac03450-9a27-4ec4-9669-9eb798da4419 (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:09.565503196Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"No ledger in /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:09.565519177Z","hostname":"buskin","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:09.565539457Z","hostname":"buskin","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:09.565546947Z","hostname":"buskin","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:645"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:09.565553697Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:09.565560847Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:09.565569348Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:09.565575808Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:09.742530464Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:09.742569535Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"c4828395-89b1-4653-93a7-55b242871c48 (zpool)"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/5050fe14-0845-40b2-872d-d70a5b2b56be): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-08-23T21:52:10.210022871Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:10.689680785Z","hostname":"buskin","pid":1562,"component":"VmmReservoirManager","sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:10.720983115Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:307","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:10.729620401Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:351","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:10.729706132Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:353","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:10.729734903Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:357","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:10.729743893Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:359","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:10.729754463Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:10.729863135Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:10.729874315Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:371"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:10.765304255Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_c4828395-89b1-4653-93a7-55b242871c48/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:10.773667967Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:414"} {"msg":"Looking for unencrypted datasets in oxp_c4828395-89b1-4653-93a7-55b242871c48","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:10.895995519Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:499"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:10.90376954Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:10.903798121Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:10.903804521Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:10.903811341Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:10.903818191Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:10.913670448Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:10.913700668Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.008929781Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:307","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.016770353Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:351","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.016819034Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:353","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.016825454Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:357","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.016863425Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:359","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.016874525Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.017038158Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.017047218Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:371"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.05373679Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_c4fe4e01-40de-4c9e-857a-e6870bd84153/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.061367099Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:414"} {"msg":"Looking for unencrypted datasets in oxp_c4fe4e01-40de-4c9e-857a-e6870bd84153","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.181106477Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:499"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.188537513Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.188584193Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.188589913Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.188596743Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.188601003Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.198644904Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.198679774Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.292085636Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:307","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","zpool":"ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.299912309Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:351","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.299951439Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:353","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.299960969Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:357","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.29997533Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:359","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.29998612Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.300110682Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.300123082Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:371"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/5050fe14-0845-40b2-872d-d70a5b2b56be): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-08-23T21:52:11.304248262Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.336233654Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_0bb3d35c-cbec-445b-9b3a-58337934f8de/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.343896404Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:414"} {"msg":"Looking for unencrypted datasets in oxp_0bb3d35c-cbec-445b-9b3a-58337934f8de","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.47418018Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:499"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.4830251Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.48307063Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.483081451Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.483087391Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.483094411Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.493153101Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.493187042Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"44b92f88-0c07-4732-953e-9cac63715495 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.590674403Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:307","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","zpool":"ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.599104326Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:351","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.599153816Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:353","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.599165816Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:357","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.599183557Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:359","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.599211697Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.59936348Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.59937525Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:371"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.638212358Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_44b92f88-0c07-4732-953e-9cac63715495/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.646740492Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:414"} {"msg":"Looking for unencrypted datasets in oxp_44b92f88-0c07-4732-953e-9cac63715495","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.77119803Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:499"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.779488901Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.779544371Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.779552631Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.779561522Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.779582922Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.790145781Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.790176811Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.884899175Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:307","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","zpool":"ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.893520772Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:351","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.893562152Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:353","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.893569552Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:357","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.893593193Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:359","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.893604633Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.893772686Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.893791126Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:371"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.930347695Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_44ff68f6-909f-4df8-8e90-d68ca84323b3/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:11.938648546Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:414"} {"msg":"Looking for unencrypted datasets in oxp_44ff68f6-909f-4df8-8e90-d68ca84323b3","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.065350231Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:499"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.073507129Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.07354974Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.07355812Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.07356581Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.07357306Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.083245914Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.083286995Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"dad70eae-b83f-4c3a-a32e-2a6a8946803c (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.181782823Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:307","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","zpool":"ZpoolName { id: dad70eae-b83f-4c3a-a32e-2a6a8946803c (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.190002362Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:351","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.190072473Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:353","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.190081203Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:357","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.190092984Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:359","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.190104734Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.190234926Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.190249176Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:371"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.226770625Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_dad70eae-b83f-4c3a-a32e-2a6a8946803c/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.234708329Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:414"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/5050fe14-0845-40b2-872d-d70a5b2b56be): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-08-23T21:52:12.306464825Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Looking for unencrypted datasets in oxp_dad70eae-b83f-4c3a-a32e-2a6a8946803c","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.358593737Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:499"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.366267197Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.366308408Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.366315218Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.366334068Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.366339268Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.376849577Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.376880247Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"ec50006e-a21c-4820-b439-48f6d6b60c97 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.472203581Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:307","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","zpool":"ZpoolName { id: ec50006e-a21c-4820-b439-48f6d6b60c97 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.480061744Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:351","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.480088855Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:353","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.480095375Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:357","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.480101875Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:359","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.480108235Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.480231967Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.480237527Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:371"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.517847554Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_ec50006e-a21c-4820-b439-48f6d6b60c97/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.526002862Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:414"} {"msg":"Looking for unencrypted datasets in oxp_ec50006e-a21c-4820-b439-48f6d6b60c97","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.658646319Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:499"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.666664734Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.666713865Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.666723986Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.666730826Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.666735296Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.676349318Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.676389319Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"e1a016b1-16ad-48d4-a26f-2e916f51bc80 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.776058207Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:307","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","zpool":"ZpoolName { id: e1a016b1-16ad-48d4-a26f-2e916f51bc80 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.784280216Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:351","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.784327257Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:353","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.784336137Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:357","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.784355158Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:359","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.784365858Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.784542621Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.784550611Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:371"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.820381758Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_e1a016b1-16ad-48d4-a26f-2e916f51bc80/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.828443784Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:414"} {"msg":"Looking for unencrypted datasets in oxp_e1a016b1-16ad-48d4-a26f-2e916f51bc80","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.953425261Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:499"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.961546379Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.961586279Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.961593429Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.961600119Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.96160618Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.971115841Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:12.971165241Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"7ac03450-9a27-4ec4-9669-9eb798da4419 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.067168327Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:307","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","zpool":"ZpoolName { id: 7ac03450-9a27-4ec4-9669-9eb798da4419 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.075238424Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:351","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.075284945Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:353","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.075297525Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:357","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.075308975Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:359","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.075321545Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.075425287Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.075434147Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:371"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.10747169Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_7ac03450-9a27-4ec4-9669-9eb798da4419/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.114341776Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:414"} {"msg":"Looking for unencrypted datasets in oxp_7ac03450-9a27-4ec4-9669-9eb798da4419","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.23857949Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:499"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.246572695Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.246630606Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Writing ledger to /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.246710538Z","hostname":"buskin","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.24686078Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 156808b0-3956-455e-a8e1-2217e0afca54 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: fe7217f3-1112-4ad5-a4fc-b58965332aab (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: dad70eae-b83f-4c3a-a32e-2a6a8946803c (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: ec50006e-a21c-4820-b439-48f6d6b60c97 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: e1a016b1-16ad-48d4-a26f-2e916f51bc80 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 7ac03450-9a27-4ec4-9669-9eb798da4419 (zpool), kind: External } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.246929291Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.246938372Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Writing ledger to /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.247063764Z","hostname":"buskin","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"physical disks ensure: Updated storage","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.247304698Z","hostname":"buskin","pid":1562,"sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:837"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.247314238Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"physical disks ensure: Propagating new generation of disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.247320208Z","hostname":"buskin","pid":1562,"sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:863","generation":"Generation(1)"} {"msg":"physical disks ensure: Updated storage monitor","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.247324988Z","hostname":"buskin","pid":1562,"sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:868"} {"msg":"physical disks ensure: Updated zone bundler","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.247328438Z","hostname":"buskin","pid":1562,"sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:873"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.247354659Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:121","gen":"Generation(1)"} {"msg":"physical disks ensure: Updated probes","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.247360819Z","hostname":"buskin","pid":1562,"sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:879"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.247402869Z","hostname":"buskin","pid":1562,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:798","gen":"Generation(1)"} {"msg":"physical disks ensure: Updated instances","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.24740718Z","hostname":"buskin","pid":1562,"sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:884"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.247486881Z","hostname":"buskin","pid":1562,"uri":"/omicron-physical-disks","method":"PUT","req_id":"8c2ceaac-a797-4d80-9c4f-a5ce33a38949","remote_addr":"[fd00:1122:3344:101::1]:59727","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":3682177,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: bd1e844a-019f-4d2c-af1c-ec9d27991436, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 07617dac-1d06-4704-81b5-9345e9dd66a9, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.27871314Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:440"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.278857732Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:36907"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.279161897Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.279170137Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/5050fe14-0845-40b2-872d-d70a5b2b56be): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-08-23T21:52:13.308130938Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.332605342Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.332641593Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.382154011Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.382195692Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.431733061Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"No ledger in /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.431768222Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.431776322Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.431791722Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.47538825Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.475420121Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.475424301Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_bd1e844a-019f-4d2c-af1c-ec9d27991436","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.52556249Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_60c8c02e-6e6f-4e1a-a162-ca6470555313","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.529296643Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_07617dac-1d06-4704-81b5-9345e9dd66a9","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.532796893Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_internal_dns_bd1e844a-019f-4d2c-af1c-ec9d27991436","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.545531588Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_60c8c02e-6e6f-4e1a-a162-ca6470555313","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.549699839Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_07617dac-1d06-4704-81b5-9345e9dd66a9","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:13.556580905Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/5050fe14-0845-40b2-872d-d70a5b2b56be): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-08-23T21:52:14.310399861Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:14.344057231Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"22.208073579s"} {"msg":"Profile for oxz_internal_dns_bd1e844a-019f-4d2c-af1c-ec9d27991436:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:15.042051172Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:15.042140783Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:1::, width: 64 }"} {"msg":"Booting oxz_internal_dns_bd1e844a-019f-4d2c-af1c-ec9d27991436 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:15.042343567Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_bd1e844a-019f-4d2c-af1c-ec9d27991436","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_internal_dns_60c8c02e-6e6f-4e1a-a162-ca6470555313:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:15.525512989Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:15.525597411Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:2::, width: 64 }"} {"msg":"Profile for oxz_internal_dns_07617dac-1d06-4704-81b5-9345e9dd66a9:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:16.007270668Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_internal_dns_bd1e844a-019f-4d2c-af1c-ec9d27991436\") failed: Property not found. retry in 72.033559ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:16.016672367Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_bd1e844a-019f-4d2c-af1c-ec9d27991436","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_internal_dns_60c8c02e-6e6f-4e1a-a162-ca6470555313 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:16.016727348Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_60c8c02e-6e6f-4e1a-a162-ca6470555313","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:16.020857878Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:3::, width: 64 }"} {"msg":"Booting oxz_internal_dns_07617dac-1d06-4704-81b5-9345e9dd66a9 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:16.020914629Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_07617dac-1d06-4704-81b5-9345e9dd66a9","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_internal_dns_bd1e844a-019f-4d2c-af1c-ec9d27991436\") failed: Property not found. retry in 118.675481ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:16.098234148Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_bd1e844a-019f-4d2c-af1c-ec9d27991436","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_60c8c02e-6e6f-4e1a-a162-ca6470555313\") failed: Property not found. retry in 73.014811ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:16.14499236Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_60c8c02e-6e6f-4e1a-a162-ca6470555313","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_07617dac-1d06-4704-81b5-9345e9dd66a9\") failed: Property not found. retry in 57.98144ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:16.151656893Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_07617dac-1d06-4704-81b5-9345e9dd66a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_07617dac-1d06-4704-81b5-9345e9dd66a9\") failed: Property not found. retry in 100.634493ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:16.286078729Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_07617dac-1d06-4704-81b5-9345e9dd66a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_bd1e844a-019f-4d2c-af1c-ec9d27991436\") failed: Property not found. retry in 200.789185ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:16.3380978Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_bd1e844a-019f-4d2c-af1c-ec9d27991436","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_60c8c02e-6e6f-4e1a-a162-ca6470555313\") failed: Property not found. retry in 89.14496ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:16.363154535Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_60c8c02e-6e6f-4e1a-a162-ca6470555313","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_07617dac-1d06-4704-81b5-9345e9dd66a9\") failed: Property not found. retry in 123.852131ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:16.461227875Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_07617dac-1d06-4704-81b5-9345e9dd66a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_60c8c02e-6e6f-4e1a-a162-ca6470555313\") failed: Property not found. retry in 224.28907ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:16.565792897Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_60c8c02e-6e6f-4e1a-a162-ca6470555313","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_bd1e844a-019f-4d2c-af1c-ec9d27991436\") failed: Property not found. retry in 414.527347ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:16.600089057Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_bd1e844a-019f-4d2c-af1c-ec9d27991436","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_07617dac-1d06-4704-81b5-9345e9dd66a9\") failed: Property not found. retry in 432.023261ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:16.61499826Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_07617dac-1d06-4704-81b5-9345e9dd66a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_60c8c02e-6e6f-4e1a-a162-ca6470555313\") failed: Property not found. retry in 538.980145ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:16.806060586Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_60c8c02e-6e6f-4e1a-a162-ca6470555313","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_bd1e844a-019f-4d2c-af1c-ec9d27991436\") failed: Property not found. retry in 1.168075977s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:17.023438556Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_bd1e844a-019f-4d2c-af1c-ec9d27991436","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_07617dac-1d06-4704-81b5-9345e9dd66a9\") failed: Property not found. retry in 490.537997ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:17.055840185Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_07617dac-1d06-4704-81b5-9345e9dd66a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_60c8c02e-6e6f-4e1a-a162-ca6470555313\") failed: Property not found. retry in 1.129984222s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:17.355778615Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_60c8c02e-6e6f-4e1a-a162-ca6470555313","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_07617dac-1d06-4704-81b5-9345e9dd66a9\") failed: Property not found. retry in 885.435162ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:17.557594493Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_07617dac-1d06-4704-81b5-9345e9dd66a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_bd1e844a-019f-4d2c-af1c-ec9d27991436\") failed: Property not found. retry in 1.119804346s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:18.200007152Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_bd1e844a-019f-4d2c-af1c-ec9d27991436","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_07617dac-1d06-4704-81b5-9345e9dd66a9\") failed: Property not found. retry in 726.686972ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:18.451847257Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_07617dac-1d06-4704-81b5-9345e9dd66a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_60c8c02e-6e6f-4e1a-a162-ca6470555313\") failed: Property not found. retry in 1.335967019s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:18.493865368Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_60c8c02e-6e6f-4e1a-a162-ca6470555313","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:19.124081811Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4206"} {"msg":"Ensuring there is a default route","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:19.124140312Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4216","gateway":"fd00:1122:3344:101::1"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:19.130547191Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4295"} {"msg":"wait for service Some(\"oxz_internal_dns_07617dac-1d06-4704-81b5-9345e9dd66a9\") failed: Property not found. retry in 1.424185158s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:19.186224053Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_07617dac-1d06-4704-81b5-9345e9dd66a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"refreshed dendrite service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:19.252414984Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4343"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:19.252448395Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4370"} {"msg":"refreshed lldpd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:19.290181194Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4383"} {"msg":"configuring MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:19.290225165Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4247"} {"msg":"wait for service Some(\"oxz_internal_dns_bd1e844a-019f-4d2c-af1c-ec9d27991436\") failed: Property not found. retry in 952.541041ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:19.32950393Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_bd1e844a-019f-4d2c-af1c-ec9d27991436","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"refreshed MGS service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:19.345812126Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4289"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:19.345842546Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4350","rack_subnet":"fd00:1122:3344::"} {"msg":"refreshed wicketd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:19.365927167Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4361"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:19.365958897Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4403"} {"msg":"refreshed mgd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:19.449113855Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4435"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:19.449153876Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4441"} {"msg":"refreshed mg-ddm service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:19.558702371Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4482"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:19.558748332Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:366"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/5050fe14-0845-40b2-872d-d70a5b2b56be): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:19.591411735Z","hostname":"buskin","pid":1562,"component":"NexusNotifierTask","sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"sled-agent/src/nexus.rs:571"} {"msg":"Failed to get switch ID from MGS (retrying in 45.859982ms)","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:19.591502207Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:385","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"Failed to get switch ID from MGS (retrying in 75.85804ms)","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:19.638754877Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:385","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"Failed to get switch ID from MGS (retrying in 185.599618ms)","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:19.714925847Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:385","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"wait for service Some(\"oxz_internal_dns_60c8c02e-6e6f-4e1a-a162-ca6470555313\") failed: Property not found. retry in 1.142735352s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:19.840062116Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_60c8c02e-6e6f-4e1a-a162-ca6470555313","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Failed to get switch ID from MGS (retrying in 321.444765ms)","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:19.901423086Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:385","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:20.199784018Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::2]:43133"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:20.200039542Z","hostname":"buskin","pid":1562,"uri":"/sled-identifiers","method":"GET","req_id":"5fdb6aac-458d-48b1-aafa-33c6a6447aff","remote_addr":"[fd00:1122:3344:101::2]:43133","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":70,"response_code":"200"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:20.224891743Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:414"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:20.259497029Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:676"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:20.259617551Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:714"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:20.26011197Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:717","uptime":"0"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:20.26012385Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:436","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.1.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n}"} {"msg":"wait for service Some(\"oxz_internal_dns_bd1e844a-019f-4d2c-af1c-ec9d27991436\") failed: Property not found. retry in 561.701388ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:20.301583272Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_bd1e844a-019f-4d2c-af1c-ec9d27991436","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:20.303074137Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:20.35696358Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:20.357036811Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:20.357049421Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:20.357056571Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:20.357064622Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:20.357072952Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:20.357079662Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:20.357086542Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:20.357094532Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:20.357100962Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:20.357109242Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:20.357115762Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:20.357128593Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:20.357133503Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:20.357138393Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:20.357143003Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:20.357147583Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:20.357152193Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:20.357157163Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:20.357162463Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:20.357167033Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:20.357171023Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:20.357174683Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"wait for service Some(\"oxz_internal_dns_07617dac-1d06-4704-81b5-9345e9dd66a9\") failed: Property not found. retry in 1.130143405s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:20.619655318Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_07617dac-1d06-4704-81b5-9345e9dd66a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/5050fe14-0845-40b2-872d-d70a5b2b56be): error trying to connect: dns error: no record found for Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:21.59402107Z","hostname":"buskin","pid":1562,"component":"NexusNotifierTask","sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"sled-agent/src/nexus.rs:571"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:21.770479598Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_internal_dns_07617dac-1d06-4704-81b5-9345e9dd66a9"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:21.770521999Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_internal_dns_bd1e844a-019f-4d2c-af1c-ec9d27991436"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:21.770528089Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_internal_dns_60c8c02e-6e6f-4e1a-a162-ca6470555313"} {"msg":"Writing ledger to /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:21.770547249Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:21.770946766Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:21.771282732Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"fe2f1be5-f1d2-4ca8-aa7f-fb678c3ec123","remote_addr":"[fd00:1122:3344:101::1]:36907","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":8492339,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:21.773193434Z","hostname":"buskin","pid":1562,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:557"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:21.825883826Z","hostname":"buskin","pid":1562,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:557"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:21.870451881Z","hostname":"buskin","pid":1562,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:557"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:21.917278484Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:593"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:21.917292944Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:260"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:21.947589327Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:21.948308479Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:302"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:21.948320479Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: bd1e844a-019f-4d2c-af1c-ec9d27991436, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 07617dac-1d06-4704-81b5-9345e9dd66a9, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d738a4fe-561c-4114-8fad-21f02e2dba0e, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ec50006e-a21c-4820-b439-48f6d6b60c97 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cfa27805-1454-49c2-aa13-243fa7ea6adb, kind: Service { id: d738a4fe-561c-4114-8fad-21f02e2dba0e }, name: Name(\"ntp-d738a4fe-561c-4114-8fad-21f02e2dba0e\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 151, 153])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:21.977374501Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:440"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:21.977457383Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:51899"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:21.977651436Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:21.977663166Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:21.999258802Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:21.999314833Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:22.020157276Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:22.020213607Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:22.038151351Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:22.038211752Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:22.038355974Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:22.038452536Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:22.049196127Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:260"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:22.083433777Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:22.084240091Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:302"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:22.084245081Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:22.114347121Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:424","port":"Port { inner: PortInner(PortData { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 151, 153]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:22.114360981Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1235","zone_type":"boundary_ntp"} {"msg":"no nat entry found for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:22.114791369Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:22.114795998Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:22.115038133Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:22.116571088Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Configuring new Omicron zone: oxz_ntp_d738a4fe-561c-4114-8fad-21f02e2dba0e","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:22.13907708Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_ntp_d738a4fe-561c-4114-8fad-21f02e2dba0e","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:22.156737809Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_ntp_d738a4fe-561c-4114-8fad-21f02e2dba0e:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:22.75612759Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_ntp_d738a4fe-561c-4114-8fad-21f02e2dba0e zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:22.756273032Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_d738a4fe-561c-4114-8fad-21f02e2dba0e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_ntp_d738a4fe-561c-4114-8fad-21f02e2dba0e\") failed: Property not found. retry in 55.297264ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:22.884050686Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_d738a4fe-561c-4114-8fad-21f02e2dba0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_d738a4fe-561c-4114-8fad-21f02e2dba0e\") failed: Property not found. retry in 106.004364ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:22.949294271Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_d738a4fe-561c-4114-8fad-21f02e2dba0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_d738a4fe-561c-4114-8fad-21f02e2dba0e\") failed: Property not found. retry in 279.248715ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:23.089572206Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_d738a4fe-561c-4114-8fad-21f02e2dba0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_d738a4fe-561c-4114-8fad-21f02e2dba0e\") failed: Property not found. retry in 320.504776ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:23.38089245Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_d738a4fe-561c-4114-8fad-21f02e2dba0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_d738a4fe-561c-4114-8fad-21f02e2dba0e\") failed: Property not found. retry in 877.343701ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:23.71214864Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_d738a4fe-561c-4114-8fad-21f02e2dba0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_d738a4fe-561c-4114-8fad-21f02e2dba0e\") failed: Property not found. retry in 1.465485883s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:24.59909671Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_d738a4fe-561c-4114-8fad-21f02e2dba0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_d738a4fe-561c-4114-8fad-21f02e2dba0e\") failed: Property not found. retry in 911.531742ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:26.074183271Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_d738a4fe-561c-4114-8fad-21f02e2dba0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_d738a4fe-561c-4114-8fad-21f02e2dba0e\") failed: Property not found. retry in 790.18764ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:26.995259469Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_d738a4fe-561c-4114-8fad-21f02e2dba0e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:27.808845527Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_ntp_d738a4fe-561c-4114-8fad-21f02e2dba0e"} {"msg":"Writing ledger to /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:27.808916279Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:27.809273685Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:27.809634771Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"c8824799-7ed1-4752-8215-0e8debe6a951","remote_addr":"[fd00:1122:3344:101::1]:51899","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":5832118,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:27.811868529Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:27.842001769Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:614"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:27.842190372Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:51874"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_d738a4fe-561c-4114-8fad-21f02e2dba0e': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-08-23T21:52:27.850962761Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3847"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:27.851096353Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"7b2de4b0-dfe9-4471-b5ef-746a673efa49","remote_addr":"[fd00:1122:3344:101::1]:51874","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8776,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:27.852827652Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:664","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:27.973659538Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:614"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:27.973798041Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:56779"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_d738a4fe-561c-4114-8fad-21f02e2dba0e': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-08-23T21:52:31.982675811Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3847"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:31.982799693Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"164406a0-0670-47a7-a8a5-154e779aa623","remote_addr":"[fd00:1122:3344:101::1]:56779","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4008874,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:31.986857072Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:664","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:32.178143321Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:614"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:32.178262103Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:40734"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_d738a4fe-561c-4114-8fad-21f02e2dba0e': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-08-23T21:52:32.187268646Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3847"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:32.187437829Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"ec00d723-1dcf-4ea8-bacb-22997d74046f","remote_addr":"[fd00:1122:3344:101::1]:40734","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":9062,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:32.189732128Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:664","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:32.400206422Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:614"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:32.400331844Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:49770"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_d738a4fe-561c-4114-8fad-21f02e2dba0e': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-08-23T21:52:32.408500093Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3847"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:32.408598544Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"367c6183-745f-483a-99e1-dc161093ef63","remote_addr":"[fd00:1122:3344:101::1]:49770","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8172,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:32.410158351Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:664","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:32.686857607Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:614"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:32.686990289Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:36231"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:33.696250271Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"9287865d-9d0a-429b-94cb-6586457e6327","remote_addr":"[fd00:1122:3344:101::1]:36231","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1009145,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:33.69797435Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:642"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:33.69798044Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:664","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:33.937824622Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:614"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:33.937956524Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:62256"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:33.946387747Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"15d430c2-53b7-4b2d-9e9c-966ed1c5108f","remote_addr":"[fd00:1122:3344:101::1]:62256","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":8300,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:33.949223175Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:642"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:33.949230125Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:664","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:34.24714606Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:614"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:34.247277502Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:34956"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:34.256078731Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"1fb21413-d749-4e92-b544-94bb98d57a49","remote_addr":"[fd00:1122:3344:101::1]:34956","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":8653,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:34.258550993Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:642"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:34.258560873Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:664","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:34.55123466Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:614"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:34.551344791Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:39525"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:34.559624932Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"5a3a608c-4632-4984-abb2-14c54268ca2d","remote_addr":"[fd00:1122:3344:101::1]:39525","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":8162,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:34.56128248Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:642"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:34.56129072Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:664","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:34.78097055Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:614"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:34.781074022Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:34858"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:34.788949086Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"231a43c9-9900-4fa4-b52c-d8f578b2ee69","remote_addr":"[fd00:1122:3344:101::1]:34858","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":7763,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:34.791327656Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:642"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:34.791335726Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:664","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.234182096Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:614"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.234297818Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:60676"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.241702494Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"4018e901-e743-4f5a-a0a0-adadc5eea5f7","remote_addr":"[fd00:1122:3344:101::1]:60676","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":7320,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.245149852Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:642"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:35.245160062Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:664","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.356591219Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:614"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.356703421Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:38029"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.376327863Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"868b043e-9838-486c-aaa0-79dacd8da26d","remote_addr":"[fd00:1122:3344:101::1]:38029","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":19517,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 2273734522, ip_addr: 135.134.111.122, stratum: 3, ref_time: 1724449955.0739956, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.379980976Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:642"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.379989965Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1266"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bd1e844a-019f-4d2c-af1c-ec9d27991436, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 07617dac-1d06-4704-81b5-9345e9dd66a9, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d738a4fe-561c-4114-8fad-21f02e2dba0e, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ec50006e-a21c-4820-b439-48f6d6b60c97 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cfa27805-1454-49c2-aa13-243fa7ea6adb, kind: Service { id: d738a4fe-561c-4114-8fad-21f02e2dba0e }, name: Name(\"ntp-d738a4fe-561c-4114-8fad-21f02e2dba0e\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 151, 153])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } } } }, OmicronZoneConfig { id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } } } }, OmicronZoneConfig { id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } } } }, OmicronZoneConfig { id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.410272178Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:440"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.41039274Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:49912"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.410590614Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.410596034Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.430298208Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.430348849Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.448964454Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.449003874Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.466921858Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.466972389Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.518643734Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.518683074Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.570429391Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.570469241Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.622505293Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.622542573Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.674113557Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.674153617Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.724669563Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.724726404Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.724901147Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.725008448Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:35.756655675Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:35.756707925Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:35.756719556Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:35.756731496Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:35.756742016Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.756849178Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"a3166e82-34f4-48ba-87b7-307d8c177a23","remote_addr":"[fd00:1122:3344:101::1]:49912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\", TimeNotSynchronized), (\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\", TimeNotSynchronized), (\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\", TimeNotSynchronized), (\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\", TimeNotSynchronized), (\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\", TimeNotSynchronized)]","latency_us":346390,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:35.757011481Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:477","retry_after":"100.509801ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a3166e82-34f4-48ba-87b7-307d8c177a23\", \"content-length\": \"124\", \"date\": \"Fri, 23 Aug 2024 21:52:35 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a3166e82-34f4-48ba-87b7-307d8c177a23\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bd1e844a-019f-4d2c-af1c-ec9d27991436, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 07617dac-1d06-4704-81b5-9345e9dd66a9, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d738a4fe-561c-4114-8fad-21f02e2dba0e, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ec50006e-a21c-4820-b439-48f6d6b60c97 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cfa27805-1454-49c2-aa13-243fa7ea6adb, kind: Service { id: d738a4fe-561c-4114-8fad-21f02e2dba0e }, name: Name(\"ntp-d738a4fe-561c-4114-8fad-21f02e2dba0e\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 151, 153])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } } } }, OmicronZoneConfig { id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } } } }, OmicronZoneConfig { id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } } } }, OmicronZoneConfig { id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.858205634Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:440"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.85854992Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.858564Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.876784489Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.876814049Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.89515513Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.895194581Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.912803429Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.912839989Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.930756453Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.930798604Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.948981961Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.949023892Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.966666351Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.966706452Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.985331837Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:35.985368508Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.00319628Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.00323723Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.003441374Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.003608007Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:36.025770146Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:36.025816707Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:36.025822967Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:36.025830237Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:36.025835938Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.025925999Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"5ade7c4a-df27-4abd-9231-9bb81b8afecc","remote_addr":"[fd00:1122:3344:101::1]:49912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\", TimeNotSynchronized), (\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\", TimeNotSynchronized), (\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\", TimeNotSynchronized), (\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\", TimeNotSynchronized), (\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\", TimeNotSynchronized)]","latency_us":177566,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:36.026057011Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:477","retry_after":"138.350117ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5ade7c4a-df27-4abd-9231-9bb81b8afecc\", \"content-length\": \"124\", \"date\": \"Fri, 23 Aug 2024 21:52:35 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5ade7c4a-df27-4abd-9231-9bb81b8afecc\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bd1e844a-019f-4d2c-af1c-ec9d27991436, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 07617dac-1d06-4704-81b5-9345e9dd66a9, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d738a4fe-561c-4114-8fad-21f02e2dba0e, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ec50006e-a21c-4820-b439-48f6d6b60c97 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cfa27805-1454-49c2-aa13-243fa7ea6adb, kind: Service { id: d738a4fe-561c-4114-8fad-21f02e2dba0e }, name: Name(\"ntp-d738a4fe-561c-4114-8fad-21f02e2dba0e\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 151, 153])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } } } }, OmicronZoneConfig { id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } } } }, OmicronZoneConfig { id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } } } }, OmicronZoneConfig { id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.165208638Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:440"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.165534493Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.165559124Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.184060797Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.184100878Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.202248365Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.202288006Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.220175999Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.220224179Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.238349026Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.238388007Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.256421172Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.256471703Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.274439758Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.274478438Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.291989925Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.292029486Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.310123792Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.310164293Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.310370216Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.310489988Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:36.344120348Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:36.344177629Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:36.344194259Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:36.344202759Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:36.344209419Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.344290811Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"6c2802de-e219-4215-a750-ed0b5f279fe9","remote_addr":"[fd00:1122:3344:101::1]:49912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\", TimeNotSynchronized), (\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\", TimeNotSynchronized), (\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\", TimeNotSynchronized), (\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\", TimeNotSynchronized), (\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\", TimeNotSynchronized)]","latency_us":178939,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:36.344418943Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:477","retry_after":"200.666262ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6c2802de-e219-4215-a750-ed0b5f279fe9\", \"content-length\": \"124\", \"date\": \"Fri, 23 Aug 2024 21:52:35 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6c2802de-e219-4215-a750-ed0b5f279fe9\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bd1e844a-019f-4d2c-af1c-ec9d27991436, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 07617dac-1d06-4704-81b5-9345e9dd66a9, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d738a4fe-561c-4114-8fad-21f02e2dba0e, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ec50006e-a21c-4820-b439-48f6d6b60c97 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cfa27805-1454-49c2-aa13-243fa7ea6adb, kind: Service { id: d738a4fe-561c-4114-8fad-21f02e2dba0e }, name: Name(\"ntp-d738a4fe-561c-4114-8fad-21f02e2dba0e\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 151, 153])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } } } }, OmicronZoneConfig { id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } } } }, OmicronZoneConfig { id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } } } }, OmicronZoneConfig { id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.546383883Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:440"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.546658288Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.546667428Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.566366032Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.566445533Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.585715769Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.58575066Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.604295664Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.604344325Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.622853338Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.622897869Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.641071787Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.641111127Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.659179203Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.659213814Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.676960094Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.677003765Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.694825287Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.694866808Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.695069061Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.695191833Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:36.727112754Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:36.727148144Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:36.727156604Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:36.727163335Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:36.727171945Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.727340798Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"aacca9b2-e399-48ff-8574-95a61894f833","remote_addr":"[fd00:1122:3344:101::1]:49912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\", TimeNotSynchronized), (\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\", TimeNotSynchronized), (\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\", TimeNotSynchronized), (\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\", TimeNotSynchronized), (\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\", TimeNotSynchronized)]","latency_us":180821,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:36.727541941Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:477","retry_after":"97.016408ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"aacca9b2-e399-48ff-8574-95a61894f833\", \"content-length\": \"124\", \"date\": \"Fri, 23 Aug 2024 21:52:36 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"aacca9b2-e399-48ff-8574-95a61894f833\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bd1e844a-019f-4d2c-af1c-ec9d27991436, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 07617dac-1d06-4704-81b5-9345e9dd66a9, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d738a4fe-561c-4114-8fad-21f02e2dba0e, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ec50006e-a21c-4820-b439-48f6d6b60c97 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cfa27805-1454-49c2-aa13-243fa7ea6adb, kind: Service { id: d738a4fe-561c-4114-8fad-21f02e2dba0e }, name: Name(\"ntp-d738a4fe-561c-4114-8fad-21f02e2dba0e\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 151, 153])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } } } }, OmicronZoneConfig { id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } } } }, OmicronZoneConfig { id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } } } }, OmicronZoneConfig { id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.825147854Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:440"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.825418748Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.825434499Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.845819764Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.845860354Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.861787784Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.861826615Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.876960061Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.877000912Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.895432954Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.895463554Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.91176189Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.911798591Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.92707174Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.92710556Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.944832431Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.944862111Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.964185378Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.964223109Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.964416452Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.964552295Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:36.99556861Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:36.99559974Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:36.99560615Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:36.99561157Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:36.995616651Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:36.995724452Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"ea5ac5f2-2d52-49ae-b911-01b8d8e8de58","remote_addr":"[fd00:1122:3344:101::1]:49912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\", TimeNotSynchronized), (\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\", TimeNotSynchronized), (\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\", TimeNotSynchronized), (\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\", TimeNotSynchronized), (\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\", TimeNotSynchronized)]","latency_us":170466,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:36.995880615Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:477","retry_after":"197.850596ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ea5ac5f2-2d52-49ae-b911-01b8d8e8de58\", \"content-length\": \"124\", \"date\": \"Fri, 23 Aug 2024 21:52:36 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ea5ac5f2-2d52-49ae-b911-01b8d8e8de58\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bd1e844a-019f-4d2c-af1c-ec9d27991436, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 07617dac-1d06-4704-81b5-9345e9dd66a9, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d738a4fe-561c-4114-8fad-21f02e2dba0e, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ec50006e-a21c-4820-b439-48f6d6b60c97 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cfa27805-1454-49c2-aa13-243fa7ea6adb, kind: Service { id: d738a4fe-561c-4114-8fad-21f02e2dba0e }, name: Name(\"ntp-d738a4fe-561c-4114-8fad-21f02e2dba0e\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 151, 153])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } } } }, OmicronZoneConfig { id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } } } }, OmicronZoneConfig { id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } } } }, OmicronZoneConfig { id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:37.195069928Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:440"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:37.195478745Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:37.195489855Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:37.214756882Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:37.214780412Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:37.233323816Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:37.233348957Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:37.250852573Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:37.250889564Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:37.269343536Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:37.269366427Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:37.287533864Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:37.287556405Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:37.305030151Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:37.305060701Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:37.324293197Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:37.324341728Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:37.343417851Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:37.343456192Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:37.343666325Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:37.343782887Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:37.375805959Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:37.37585102Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:37.37585921Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:37.37586882Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:37.375878621Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:37.375978832Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"b08cb024-880d-4f76-bd10-d079794a1295","remote_addr":"[fd00:1122:3344:101::1]:49912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\", TimeNotSynchronized), (\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\", TimeNotSynchronized), (\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\", TimeNotSynchronized), (\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\", TimeNotSynchronized), (\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\", TimeNotSynchronized)]","latency_us":180717,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:37.376132715Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:477","retry_after":"327.839309ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b08cb024-880d-4f76-bd10-d079794a1295\", \"content-length\": \"124\", \"date\": \"Fri, 23 Aug 2024 21:52:36 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b08cb024-880d-4f76-bd10-d079794a1295\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bd1e844a-019f-4d2c-af1c-ec9d27991436, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 07617dac-1d06-4704-81b5-9345e9dd66a9, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d738a4fe-561c-4114-8fad-21f02e2dba0e, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ec50006e-a21c-4820-b439-48f6d6b60c97 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cfa27805-1454-49c2-aa13-243fa7ea6adb, kind: Service { id: d738a4fe-561c-4114-8fad-21f02e2dba0e }, name: Name(\"ntp-d738a4fe-561c-4114-8fad-21f02e2dba0e\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 151, 153])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } } } }, OmicronZoneConfig { id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } } } }, OmicronZoneConfig { id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } } } }, OmicronZoneConfig { id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:37.705164757Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:440"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:37.705456002Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:37.705467192Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:37.725366009Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:37.72540851Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:37.744024525Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:37.744059036Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:37.76260215Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:37.76263744Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:37.781129143Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:37.781165034Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:37.799876551Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:37.799915922Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:37.818066739Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:37.818109939Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:37.837261714Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:37.837299364Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:37.855770807Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:37.855818648Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:37.856011271Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:37.856135814Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:37.888600993Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:37.888634394Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:37.888639624Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:37.888644914Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:37.888649064Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:37.888753486Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"098dace5-b0fc-48ca-8f26-a8cc48721a2c","remote_addr":"[fd00:1122:3344:101::1]:49912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\", TimeNotSynchronized), (\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\", TimeNotSynchronized), (\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\", TimeNotSynchronized), (\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\", TimeNotSynchronized), (\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\", TimeNotSynchronized)]","latency_us":183467,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:37.888911709Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:477","retry_after":"232.83139ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"098dace5-b0fc-48ca-8f26-a8cc48721a2c\", \"content-length\": \"124\", \"date\": \"Fri, 23 Aug 2024 21:52:37 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"098dace5-b0fc-48ca-8f26-a8cc48721a2c\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bd1e844a-019f-4d2c-af1c-ec9d27991436, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 07617dac-1d06-4704-81b5-9345e9dd66a9, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d738a4fe-561c-4114-8fad-21f02e2dba0e, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ec50006e-a21c-4820-b439-48f6d6b60c97 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cfa27805-1454-49c2-aa13-243fa7ea6adb, kind: Service { id: d738a4fe-561c-4114-8fad-21f02e2dba0e }, name: Name(\"ntp-d738a4fe-561c-4114-8fad-21f02e2dba0e\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 151, 153])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } } } }, OmicronZoneConfig { id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } } } }, OmicronZoneConfig { id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } } } }, OmicronZoneConfig { id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:38.12362668Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:440"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:38.123979768Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:38.123993549Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:38.143650193Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:38.14369779Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:38.160221265Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:38.160246015Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:38.177642503Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:38.177668228Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:38.19517282Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:38.195203168Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:38.212859614Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:38.212891461Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:38.230502465Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:38.2305339Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:38.249682926Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:38.249712618Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:38.268046337Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:38.268080969Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:38.268256171Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:38.268379737Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:38.299296675Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:38.299331015Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:38.299339678Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:38.299345932Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:38.299353497Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:38.299470134Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"be1db539-4b81-42d5-b1b5-d202c5d3a2f2","remote_addr":"[fd00:1122:3344:101::1]:49912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\", TimeNotSynchronized), (\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\", TimeNotSynchronized), (\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\", TimeNotSynchronized), (\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\", TimeNotSynchronized), (\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\", TimeNotSynchronized)]","latency_us":175222,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:38.299600504Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:477","retry_after":"260.327947ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"be1db539-4b81-42d5-b1b5-d202c5d3a2f2\", \"content-length\": \"124\", \"date\": \"Fri, 23 Aug 2024 21:52:37 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"be1db539-4b81-42d5-b1b5-d202c5d3a2f2\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bd1e844a-019f-4d2c-af1c-ec9d27991436, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 07617dac-1d06-4704-81b5-9345e9dd66a9, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d738a4fe-561c-4114-8fad-21f02e2dba0e, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ec50006e-a21c-4820-b439-48f6d6b60c97 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cfa27805-1454-49c2-aa13-243fa7ea6adb, kind: Service { id: d738a4fe-561c-4114-8fad-21f02e2dba0e }, name: Name(\"ntp-d738a4fe-561c-4114-8fad-21f02e2dba0e\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 151, 153])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } } } }, OmicronZoneConfig { id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } } } }, OmicronZoneConfig { id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } } } }, OmicronZoneConfig { id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:38.561771744Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:440"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:38.562886204Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:38.562898232Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:38.582331516Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:38.582362323Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:38.600940354Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:38.600980349Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:38.61829234Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:38.61832354Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:38.636561935Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:38.63659577Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:38.65524845Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:38.655285144Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:38.673734373Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:38.673838205Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:38.691321305Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:38.691353452Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:38.709526226Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:38.70957794Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:38.709762059Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:38.709940458Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:38.742027516Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:38.742057675Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:38.742063853Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:38.742086917Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:38.742094089Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:38.742225462Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"e9cdb75c-5ad0-4623-bec2-d85790da21c9","remote_addr":"[fd00:1122:3344:101::1]:49912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\", TimeNotSynchronized), (\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\", TimeNotSynchronized), (\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\", TimeNotSynchronized), (\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\", TimeNotSynchronized), (\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\", TimeNotSynchronized)]","latency_us":179785,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:38.742400562Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:477","retry_after":"468.105964ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e9cdb75c-5ad0-4623-bec2-d85790da21c9\", \"content-length\": \"124\", \"date\": \"Fri, 23 Aug 2024 21:52:38 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e9cdb75c-5ad0-4623-bec2-d85790da21c9\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bd1e844a-019f-4d2c-af1c-ec9d27991436, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 07617dac-1d06-4704-81b5-9345e9dd66a9, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d738a4fe-561c-4114-8fad-21f02e2dba0e, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ec50006e-a21c-4820-b439-48f6d6b60c97 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cfa27805-1454-49c2-aa13-243fa7ea6adb, kind: Service { id: d738a4fe-561c-4114-8fad-21f02e2dba0e }, name: Name(\"ntp-d738a4fe-561c-4114-8fad-21f02e2dba0e\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 151, 153])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } } } }, OmicronZoneConfig { id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } } } }, OmicronZoneConfig { id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } } } }, OmicronZoneConfig { id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:39.212058873Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:440"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:39.212475159Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:39.212488238Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:39.23270905Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:39.232744226Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:39.249913915Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:39.24994685Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:39.266186686Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:39.266211334Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:39.285222568Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:39.285250046Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:39.301859566Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:39.301881775Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:39.317403176Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:39.317430111Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:39.335660716Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:39.335684997Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:39.354455016Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:39.35448285Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:39.354714346Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:39.35481527Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:39.38765814Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:39.387773879Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:39.387781116Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:39.387790407Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:39.387797663Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:39.387933913Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"3ea2b9fc-5259-4977-879c-21456a4a5a28","remote_addr":"[fd00:1122:3344:101::1]:49912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\", TimeNotSynchronized), (\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\", TimeNotSynchronized), (\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\", TimeNotSynchronized), (\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\", TimeNotSynchronized), (\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\", TimeNotSynchronized)]","latency_us":175183,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:39.388129075Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:477","retry_after":"454.172644ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3ea2b9fc-5259-4977-879c-21456a4a5a28\", \"content-length\": \"124\", \"date\": \"Fri, 23 Aug 2024 21:52:39 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3ea2b9fc-5259-4977-879c-21456a4a5a28\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bd1e844a-019f-4d2c-af1c-ec9d27991436, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 07617dac-1d06-4704-81b5-9345e9dd66a9, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d738a4fe-561c-4114-8fad-21f02e2dba0e, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ec50006e-a21c-4820-b439-48f6d6b60c97 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cfa27805-1454-49c2-aa13-243fa7ea6adb, kind: Service { id: d738a4fe-561c-4114-8fad-21f02e2dba0e }, name: Name(\"ntp-d738a4fe-561c-4114-8fad-21f02e2dba0e\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 151, 153])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } } } }, OmicronZoneConfig { id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } } } }, OmicronZoneConfig { id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } } } }, OmicronZoneConfig { id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:39.843866127Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:440"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:39.844334452Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:39.844345627Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:39.864819369Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:39.864845826Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:39.88349671Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:39.883534051Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:39.901183871Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:39.901222468Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:39.919026687Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:39.919056744Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:39.935690954Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:39.93572673Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:39.953605313Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:39.953638173Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:39.971779104Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:39.971811037Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:39.988977984Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:39.989015504Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:39.989172013Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:39.989267142Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:40.020939063Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:40.020983004Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:40.020992398Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:40.021001436Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:40.021008514Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:40.02112395Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"cb2750a8-5b34-4caa-aace-9a428d34ddba","remote_addr":"[fd00:1122:3344:101::1]:49912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\", TimeNotSynchronized), (\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\", TimeNotSynchronized), (\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\", TimeNotSynchronized), (\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\", TimeNotSynchronized), (\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\", TimeNotSynchronized)]","latency_us":176549,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:40.021270794Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:477","retry_after":"367.997458ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"cb2750a8-5b34-4caa-aace-9a428d34ddba\", \"content-length\": \"124\", \"date\": \"Fri, 23 Aug 2024 21:52:39 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"cb2750a8-5b34-4caa-aace-9a428d34ddba\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bd1e844a-019f-4d2c-af1c-ec9d27991436, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 07617dac-1d06-4704-81b5-9345e9dd66a9, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d738a4fe-561c-4114-8fad-21f02e2dba0e, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ec50006e-a21c-4820-b439-48f6d6b60c97 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cfa27805-1454-49c2-aa13-243fa7ea6adb, kind: Service { id: d738a4fe-561c-4114-8fad-21f02e2dba0e }, name: Name(\"ntp-d738a4fe-561c-4114-8fad-21f02e2dba0e\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 151, 153])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } } } }, OmicronZoneConfig { id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } } } }, OmicronZoneConfig { id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } } } }, OmicronZoneConfig { id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:40.391703025Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:440"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:40.392789373Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:40.392799826Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:40.412244987Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:40.412272644Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:40.430657081Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:40.430696204Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:40.448291104Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:40.448343905Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:40.46730614Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:40.467340481Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:40.486980939Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:40.487011745Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:40.504976054Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:40.505018336Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:40.523715761Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:40.523819448Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:40.541306673Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:40.541352385Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:40.541551064Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:40.541671984Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:40.572325146Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:40.572361512Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:40.572368815Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:40.572374543Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:40.572381809Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:40.572518884Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"400fc871-a220-4814-b3ae-a4181b0d9eed","remote_addr":"[fd00:1122:3344:101::1]:49912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\", TimeNotSynchronized), (\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\", TimeNotSynchronized), (\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\", TimeNotSynchronized), (\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\", TimeNotSynchronized), (\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\", TimeNotSynchronized)]","latency_us":180176,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:40.572689174Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:477","retry_after":"772.823381ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"400fc871-a220-4814-b3ae-a4181b0d9eed\", \"content-length\": \"124\", \"date\": \"Fri, 23 Aug 2024 21:52:39 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"400fc871-a220-4814-b3ae-a4181b0d9eed\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bd1e844a-019f-4d2c-af1c-ec9d27991436, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 07617dac-1d06-4704-81b5-9345e9dd66a9, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d738a4fe-561c-4114-8fad-21f02e2dba0e, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ec50006e-a21c-4820-b439-48f6d6b60c97 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cfa27805-1454-49c2-aa13-243fa7ea6adb, kind: Service { id: d738a4fe-561c-4114-8fad-21f02e2dba0e }, name: Name(\"ntp-d738a4fe-561c-4114-8fad-21f02e2dba0e\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 151, 153])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } } } }, OmicronZoneConfig { id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } } } }, OmicronZoneConfig { id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } } } }, OmicronZoneConfig { id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:41.348338996Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:440"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:41.348690611Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:41.348705123Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:41.369916877Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:41.369944664Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:41.389013327Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:41.389035977Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:41.408100955Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:41.408197647Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:41.428053185Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:41.428078142Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:41.446214316Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:41.44630467Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:41.464669262Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:41.464702451Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:41.482429502Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:41.482462147Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:41.501088817Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:41.501129205Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:41.501310802Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:41.501500723Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:41.532287755Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:41.532330628Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:41.532338128Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:41.532344381Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:41.532350738Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:41.532445071Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"d021995e-f5d0-41cd-ba8b-e800418691d8","remote_addr":"[fd00:1122:3344:101::1]:49912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\", TimeNotSynchronized), (\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\", TimeNotSynchronized), (\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\", TimeNotSynchronized), (\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\", TimeNotSynchronized), (\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\", TimeNotSynchronized)]","latency_us":183488,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:41.532659633Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:477","retry_after":"1.140448288s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d021995e-f5d0-41cd-ba8b-e800418691d8\", \"content-length\": \"124\", \"date\": \"Fri, 23 Aug 2024 21:52:41 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d021995e-f5d0-41cd-ba8b-e800418691d8\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bd1e844a-019f-4d2c-af1c-ec9d27991436, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 07617dac-1d06-4704-81b5-9345e9dd66a9, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d738a4fe-561c-4114-8fad-21f02e2dba0e, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ec50006e-a21c-4820-b439-48f6d6b60c97 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cfa27805-1454-49c2-aa13-243fa7ea6adb, kind: Service { id: d738a4fe-561c-4114-8fad-21f02e2dba0e }, name: Name(\"ntp-d738a4fe-561c-4114-8fad-21f02e2dba0e\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 151, 153])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } } } }, OmicronZoneConfig { id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } } } }, OmicronZoneConfig { id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } } } }, OmicronZoneConfig { id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:42.677146215Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:440"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:42.677689185Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:42.6777128Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:42.698834014Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:42.698871121Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:42.717778527Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:42.717814228Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:42.736594722Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:42.736623606Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:42.755236686Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:42.75526588Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:42.774086162Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:42.77412251Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:42.792378609Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:42.792416288Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:42.810396673Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:42.810432102Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:42.828957116Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:42.829069722Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:42.829268766Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:42.829411953Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:42.861271389Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:42.861317018Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:42.861322212Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:42.861329477Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:42.861336781Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:42.861451777Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"b1ca3424-6564-4fd8-80a5-bab9b27401c5","remote_addr":"[fd00:1122:3344:101::1]:49912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\", TimeNotSynchronized), (\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\", TimeNotSynchronized), (\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\", TimeNotSynchronized), (\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\", TimeNotSynchronized), (\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\", TimeNotSynchronized)]","latency_us":183582,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:42.861623942Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:477","retry_after":"1.22132017s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b1ca3424-6564-4fd8-80a5-bab9b27401c5\", \"content-length\": \"124\", \"date\": \"Fri, 23 Aug 2024 21:52:42 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b1ca3424-6564-4fd8-80a5-bab9b27401c5\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bd1e844a-019f-4d2c-af1c-ec9d27991436, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 07617dac-1d06-4704-81b5-9345e9dd66a9, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d738a4fe-561c-4114-8fad-21f02e2dba0e, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ec50006e-a21c-4820-b439-48f6d6b60c97 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cfa27805-1454-49c2-aa13-243fa7ea6adb, kind: Service { id: d738a4fe-561c-4114-8fad-21f02e2dba0e }, name: Name(\"ntp-d738a4fe-561c-4114-8fad-21f02e2dba0e\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 151, 153])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } } } }, OmicronZoneConfig { id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } } } }, OmicronZoneConfig { id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } } } }, OmicronZoneConfig { id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:44.087766608Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:440"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:44.088093126Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:44.088100945Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:44.10856197Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:44.108591229Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:44.127648281Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:44.127741864Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:44.146947204Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:44.14698297Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:44.166185539Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:44.166209633Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:44.184998556Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:44.18505267Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:44.20372314Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:44.203745705Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:44.221757302Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:44.221791427Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:44.240325167Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:44.240362612Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:44.240552441Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:44.240688699Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:44.272844518Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:44.272896906Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:44.272907351Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:44.27291484Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:44.272935869Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:44.273026209Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"41698440-58c0-49bd-875d-c9ccc53161e7","remote_addr":"[fd00:1122:3344:101::1]:49912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\", TimeNotSynchronized), (\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\", TimeNotSynchronized), (\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\", TimeNotSynchronized), (\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\", TimeNotSynchronized), (\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\", TimeNotSynchronized)]","latency_us":184655,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:44.273249126Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:477","retry_after":"1.363105965s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"41698440-58c0-49bd-875d-c9ccc53161e7\", \"content-length\": \"124\", \"date\": \"Fri, 23 Aug 2024 21:52:44 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"41698440-58c0-49bd-875d-c9ccc53161e7\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bd1e844a-019f-4d2c-af1c-ec9d27991436, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 07617dac-1d06-4704-81b5-9345e9dd66a9, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d738a4fe-561c-4114-8fad-21f02e2dba0e, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ec50006e-a21c-4820-b439-48f6d6b60c97 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cfa27805-1454-49c2-aa13-243fa7ea6adb, kind: Service { id: d738a4fe-561c-4114-8fad-21f02e2dba0e }, name: Name(\"ntp-d738a4fe-561c-4114-8fad-21f02e2dba0e\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 151, 153])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } } } }, OmicronZoneConfig { id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } } } }, OmicronZoneConfig { id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } } } }, OmicronZoneConfig { id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:45.641564395Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:440"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:45.642055778Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:45.642072315Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:45.662647517Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:45.662684277Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:45.68185749Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:45.681895675Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:45.699240744Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:45.699281405Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:45.716518534Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:45.716550785Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:45.733538452Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:45.733568237Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:45.752080309Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:45.752194881Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:45.769718971Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:45.769752431Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:45.786851522Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:45.786893598Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:45.78709792Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:45.787259842Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:45.818667502Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:45.81870866Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:45.818717669Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:45.81872666Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:45.818735491Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:45.818851941Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"0d6c0cf5-59cc-4203-ab46-f59eef1a4754","remote_addr":"[fd00:1122:3344:101::1]:49912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\", TimeNotSynchronized), (\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\", TimeNotSynchronized), (\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\", TimeNotSynchronized), (\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\", TimeNotSynchronized), (\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\", TimeNotSynchronized)]","latency_us":176611,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:45.819006564Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:477","retry_after":"1.13880456s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0d6c0cf5-59cc-4203-ab46-f59eef1a4754\", \"content-length\": \"124\", \"date\": \"Fri, 23 Aug 2024 21:52:45 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0d6c0cf5-59cc-4203-ab46-f59eef1a4754\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bd1e844a-019f-4d2c-af1c-ec9d27991436, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 07617dac-1d06-4704-81b5-9345e9dd66a9, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d738a4fe-561c-4114-8fad-21f02e2dba0e, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ec50006e-a21c-4820-b439-48f6d6b60c97 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cfa27805-1454-49c2-aa13-243fa7ea6adb, kind: Service { id: d738a4fe-561c-4114-8fad-21f02e2dba0e }, name: Name(\"ntp-d738a4fe-561c-4114-8fad-21f02e2dba0e\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 151, 153])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } } } }, OmicronZoneConfig { id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } } } }, OmicronZoneConfig { id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } } } }, OmicronZoneConfig { id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:46.961306216Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:440"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:46.961587043Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:46.961594028Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:46.982008207Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:46.982044142Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:47.001731409Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:47.001769941Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:47.022116389Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:47.022152962Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:47.041702208Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:47.041740233Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:47.060203998Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:47.060240514Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:47.078628642Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:47.078659917Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:47.096697298Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:47.096805559Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:47.114199626Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:47.114245442Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:47.114432458Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:47.11454706Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:47.147066416Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:47.147093942Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:47.147098911Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:47.147104658Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:47.147109608Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:47.147215969Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"3e2faf79-75b1-492b-8228-9b6afa614507","remote_addr":"[fd00:1122:3344:101::1]:49912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\", TimeNotSynchronized), (\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\", TimeNotSynchronized), (\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\", TimeNotSynchronized), (\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\", TimeNotSynchronized), (\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\", TimeNotSynchronized)]","latency_us":185298,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:47.147371588Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:477","retry_after":"1.673425029s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3e2faf79-75b1-492b-8228-9b6afa614507\", \"content-length\": \"124\", \"date\": \"Fri, 23 Aug 2024 21:52:47 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3e2faf79-75b1-492b-8228-9b6afa614507\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bd1e844a-019f-4d2c-af1c-ec9d27991436, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 07617dac-1d06-4704-81b5-9345e9dd66a9, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d738a4fe-561c-4114-8fad-21f02e2dba0e, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ec50006e-a21c-4820-b439-48f6d6b60c97 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cfa27805-1454-49c2-aa13-243fa7ea6adb, kind: Service { id: d738a4fe-561c-4114-8fad-21f02e2dba0e }, name: Name(\"ntp-d738a4fe-561c-4114-8fad-21f02e2dba0e\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 151, 153])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } } } }, OmicronZoneConfig { id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } } } }, OmicronZoneConfig { id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } } } }, OmicronZoneConfig { id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:48.826303368Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:440"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:48.826737736Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:48.826750768Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:48.847889917Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:48.847909108Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:48.866481993Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:48.866509622Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:48.885365944Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:48.885397912Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:48.904086924Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:48.904109217Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:48.923158214Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:48.923179796Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:48.9417014Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:48.94172824Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:48.960921097Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:48.960937776Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:48.979111536Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:48.979135498Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:48.979302639Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:48.979390897Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:49.013335145Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:49.01338638Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:49.01339839Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:49.013406058Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:49.013414815Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:49.013498348Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"40c8589a-cf01-4017-abdd-e1db14a621d0","remote_addr":"[fd00:1122:3344:101::1]:49912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\", TimeNotSynchronized), (\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\", TimeNotSynchronized), (\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\", TimeNotSynchronized), (\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\", TimeNotSynchronized), (\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\", TimeNotSynchronized)]","latency_us":186492,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:49.013628578Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:477","retry_after":"2.686894919s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"40c8589a-cf01-4017-abdd-e1db14a621d0\", \"content-length\": \"124\", \"date\": \"Fri, 23 Aug 2024 21:52:48 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"40c8589a-cf01-4017-abdd-e1db14a621d0\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:49.106072169Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:46498"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:49.106295648Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"af3b3144-aa80-46cc-9c46-32a20461c500","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:46498","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":59,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:51.411181762Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"128.778684ms"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bd1e844a-019f-4d2c-af1c-ec9d27991436, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 07617dac-1d06-4704-81b5-9345e9dd66a9, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d738a4fe-561c-4114-8fad-21f02e2dba0e, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ec50006e-a21c-4820-b439-48f6d6b60c97 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cfa27805-1454-49c2-aa13-243fa7ea6adb, kind: Service { id: d738a4fe-561c-4114-8fad-21f02e2dba0e }, name: Name(\"ntp-d738a4fe-561c-4114-8fad-21f02e2dba0e\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 151, 153])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } } } }, OmicronZoneConfig { id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } } } }, OmicronZoneConfig { id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } } } }, OmicronZoneConfig { id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:51.709016401Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:440"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:51.709232058Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:51.709241526Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:51.730755812Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:51.730807974Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:51.750307416Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:51.750355088Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:51.768625925Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:51.768669247Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:51.787470295Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:51.787575036Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:51.806954121Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:51.806991697Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:51.828182951Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:51.828224108Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:51.846258292Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:51.846300762Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:51.86422766Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:51.864264008Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:51.864451127Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:51.864572358Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:51.896466247Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:51.896500429Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:51.896507489Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:51.896515879Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:51.896522451Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:51.896656076Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"6b95d01c-0959-462f-905e-5472a0179ae4","remote_addr":"[fd00:1122:3344:101::1]:49912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\", TimeNotSynchronized), (\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\", TimeNotSynchronized), (\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\", TimeNotSynchronized), (\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\", TimeNotSynchronized), (\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\", TimeNotSynchronized)]","latency_us":187050,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:51.896900507Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:477","retry_after":"1.578725118s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6b95d01c-0959-462f-905e-5472a0179ae4\", \"content-length\": \"124\", \"date\": \"Fri, 23 Aug 2024 21:52:51 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6b95d01c-0959-462f-905e-5472a0179ae4\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bd1e844a-019f-4d2c-af1c-ec9d27991436, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 07617dac-1d06-4704-81b5-9345e9dd66a9, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d738a4fe-561c-4114-8fad-21f02e2dba0e, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ec50006e-a21c-4820-b439-48f6d6b60c97 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cfa27805-1454-49c2-aa13-243fa7ea6adb, kind: Service { id: d738a4fe-561c-4114-8fad-21f02e2dba0e }, name: Name(\"ntp-d738a4fe-561c-4114-8fad-21f02e2dba0e\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 151, 153])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } } } }, OmicronZoneConfig { id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } } } }, OmicronZoneConfig { id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } } } }, OmicronZoneConfig { id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:53.48187527Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:440"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:53.482334676Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:53.482356099Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:53.503122814Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:53.503244734Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:53.522082077Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:53.522128455Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:53.542102579Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:53.542151442Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:53.561755554Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:53.561799617Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:53.580102188Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:53.580158195Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:53.598507844Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:53.598554701Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:53.616418902Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:53.616462834Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:53.63394943Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:53.634005353Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:53.634207885Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:53.634331702Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:53.666360374Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:53.666400106Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:53.666409491Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:53.666417272Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:53.666426497Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:53.666542346Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"fcd0a256-3931-4862-9407-d53330fc5d0f","remote_addr":"[fd00:1122:3344:101::1]:49912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\", TimeNotSynchronized), (\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\", TimeNotSynchronized), (\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\", TimeNotSynchronized), (\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\", TimeNotSynchronized), (\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\", TimeNotSynchronized)]","latency_us":184011,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:53.666790288Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:477","retry_after":"3.231456905s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fcd0a256-3931-4862-9407-d53330fc5d0f\", \"content-length\": \"124\", \"date\": \"Fri, 23 Aug 2024 21:52:53 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"fcd0a256-3931-4862-9407-d53330fc5d0f\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bd1e844a-019f-4d2c-af1c-ec9d27991436, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 07617dac-1d06-4704-81b5-9345e9dd66a9, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d738a4fe-561c-4114-8fad-21f02e2dba0e, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ec50006e-a21c-4820-b439-48f6d6b60c97 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cfa27805-1454-49c2-aa13-243fa7ea6adb, kind: Service { id: d738a4fe-561c-4114-8fad-21f02e2dba0e }, name: Name(\"ntp-d738a4fe-561c-4114-8fad-21f02e2dba0e\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 151, 153])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } } } }, OmicronZoneConfig { id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } } } }, OmicronZoneConfig { id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } } } }, OmicronZoneConfig { id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:56.907993936Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:440"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:56.908298526Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:56.908308726Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:56.9284618Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:56.928579209Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:56.948203349Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:56.948237437Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:56.966717314Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:56.966763703Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:56.984920252Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:56.984957866Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:57.002942615Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:57.002983828Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:57.02029306Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:57.020331489Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:57.039178825Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:57.039223291Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:57.056609487Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:57.056651065Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:57.056922925Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:57.057063009Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:57.089567962Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:57.089608313Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:57.089613929Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:57.089619545Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:57.089626379Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:52:57.089721865Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"72f608d6-a1cb-4fae-a54e-acf069e50970","remote_addr":"[fd00:1122:3344:101::1]:49912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\", TimeNotSynchronized), (\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\", TimeNotSynchronized), (\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\", TimeNotSynchronized), (\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\", TimeNotSynchronized), (\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\", TimeNotSynchronized)]","latency_us":181122,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:52:57.089852367Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:477","retry_after":"3.116733437s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"72f608d6-a1cb-4fae-a54e-acf069e50970\", \"content-length\": \"124\", \"date\": \"Fri, 23 Aug 2024 21:52:56 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"72f608d6-a1cb-4fae-a54e-acf069e50970\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bd1e844a-019f-4d2c-af1c-ec9d27991436, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 07617dac-1d06-4704-81b5-9345e9dd66a9, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d738a4fe-561c-4114-8fad-21f02e2dba0e, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ec50006e-a21c-4820-b439-48f6d6b60c97 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cfa27805-1454-49c2-aa13-243fa7ea6adb, kind: Service { id: d738a4fe-561c-4114-8fad-21f02e2dba0e }, name: Name(\"ntp-d738a4fe-561c-4114-8fad-21f02e2dba0e\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 151, 153])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } } } }, OmicronZoneConfig { id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } } } }, OmicronZoneConfig { id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } } } }, OmicronZoneConfig { id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:00.215881353Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:440"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:00.216168102Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:00.21620685Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:00.237398837Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:00.237438673Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:00.256446988Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:00.256471692Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:00.275354696Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:00.275383571Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:00.29405368Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:00.294084824Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:00.313400132Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:00.313433648Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:00.331299733Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:00.331323828Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:00.349335322Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:00.349368276Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:00.367076943Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:00.367111022Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:00.367278218Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:00.367367985Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:00.399184263Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:00.399230744Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:00.399262949Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:00.39927073Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:00.399279608Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:00.399340828Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"6ffb2182-e4ac-4007-a935-58fd89bf0838","remote_addr":"[fd00:1122:3344:101::1]:49912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\", TimeNotSynchronized), (\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\", TimeNotSynchronized), (\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\", TimeNotSynchronized), (\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\", TimeNotSynchronized), (\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\", TimeNotSynchronized)]","latency_us":182824,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:00.399458279Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:477","retry_after":"3.046296379s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6ffb2182-e4ac-4007-a935-58fd89bf0838\", \"content-length\": \"124\", \"date\": \"Fri, 23 Aug 2024 21:53:00 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6ffb2182-e4ac-4007-a935-58fd89bf0838\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bd1e844a-019f-4d2c-af1c-ec9d27991436, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 07617dac-1d06-4704-81b5-9345e9dd66a9, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d738a4fe-561c-4114-8fad-21f02e2dba0e, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ec50006e-a21c-4820-b439-48f6d6b60c97 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cfa27805-1454-49c2-aa13-243fa7ea6adb, kind: Service { id: d738a4fe-561c-4114-8fad-21f02e2dba0e }, name: Name(\"ntp-d738a4fe-561c-4114-8fad-21f02e2dba0e\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 151, 153])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } } } }, OmicronZoneConfig { id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } } } }, OmicronZoneConfig { id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } } } }, OmicronZoneConfig { id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:03.455727005Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:440"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:03.456161057Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:03.456186322Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:03.476706606Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:03.47678247Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:03.493065927Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:03.493104665Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:03.508331457Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:03.508376073Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:03.523821442Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:03.523861287Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:03.541256602Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:03.541302081Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:03.559485456Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:03.559523614Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:03.577462554Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:03.577498011Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:03.594227739Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:03.594278767Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:03.594491358Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:03.594615083Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:03.625886859Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:03.625932909Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:03.625942219Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:03.625949457Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:03.625972894Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:03.626098137Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"21b7ee46-6bd2-4d3f-b620-8b0e0dce8de9","remote_addr":"[fd00:1122:3344:101::1]:49912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\", TimeNotSynchronized), (\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\", TimeNotSynchronized), (\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\", TimeNotSynchronized), (\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\", TimeNotSynchronized), (\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\", TimeNotSynchronized)]","latency_us":169636,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:03.626235624Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:477","retry_after":"3.989540301s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"21b7ee46-6bd2-4d3f-b620-8b0e0dce8de9\", \"content-length\": \"124\", \"date\": \"Fri, 23 Aug 2024 21:53:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"21b7ee46-6bd2-4d3f-b620-8b0e0dce8de9\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:06.615962397Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"444.168388ms"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bd1e844a-019f-4d2c-af1c-ec9d27991436, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 07617dac-1d06-4704-81b5-9345e9dd66a9, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d738a4fe-561c-4114-8fad-21f02e2dba0e, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ec50006e-a21c-4820-b439-48f6d6b60c97 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cfa27805-1454-49c2-aa13-243fa7ea6adb, kind: Service { id: d738a4fe-561c-4114-8fad-21f02e2dba0e }, name: Name(\"ntp-d738a4fe-561c-4114-8fad-21f02e2dba0e\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 151, 153])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } } } }, OmicronZoneConfig { id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } } } }, OmicronZoneConfig { id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } } } }, OmicronZoneConfig { id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:07.627758605Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:440"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:07.628069504Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:07.628083332Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:07.649483211Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:07.64955988Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:07.670000869Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:07.670039429Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:07.689268557Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:07.689310491Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:07.708444103Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:07.708478679Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:07.726604658Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:07.726639881Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:07.745708981Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:07.745744814Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:07.763155624Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:07.76319424Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:07.780757133Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:07.780808697Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:07.781090772Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:07.781218959Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:07.814480042Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:07.814528493Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:07.814536199Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:07.814544711Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:07.814550899Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:07.814641707Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"f8f13418-f7b0-4982-beba-49f0c6e5b9df","remote_addr":"[fd00:1122:3344:101::1]:49912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\", TimeNotSynchronized), (\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\", TimeNotSynchronized), (\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\", TimeNotSynchronized), (\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\", TimeNotSynchronized), (\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\", TimeNotSynchronized)]","latency_us":186275,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:07.814752484Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:477","retry_after":"3.569827878s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f8f13418-f7b0-4982-beba-49f0c6e5b9df\", \"content-length\": \"124\", \"date\": \"Fri, 23 Aug 2024 21:53:07 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f8f13418-f7b0-4982-beba-49f0c6e5b9df\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bd1e844a-019f-4d2c-af1c-ec9d27991436, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 07617dac-1d06-4704-81b5-9345e9dd66a9, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d738a4fe-561c-4114-8fad-21f02e2dba0e, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ec50006e-a21c-4820-b439-48f6d6b60c97 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cfa27805-1454-49c2-aa13-243fa7ea6adb, kind: Service { id: d738a4fe-561c-4114-8fad-21f02e2dba0e }, name: Name(\"ntp-d738a4fe-561c-4114-8fad-21f02e2dba0e\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 151, 153])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } } } }, OmicronZoneConfig { id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } } } }, OmicronZoneConfig { id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } } } }, OmicronZoneConfig { id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:11.395413753Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:440"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:11.395738762Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:11.395748634Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:11.416640289Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:11.416675137Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:11.436243348Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:11.436298661Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:11.455524767Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:11.45556402Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:11.473697339Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:11.47372792Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:11.492723122Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:11.492757885Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:11.511542619Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:11.511582819Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:11.530080877Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:11.530115209Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:11.548170805Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:11.548219443Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:11.548492051Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:11.548611932Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:11.581169753Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:11.581206851Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:11.581216132Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:11.581223501Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:11.581233241Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:11.581417148Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"ed4ed590-a0d9-4bb2-bfca-f10a930a9a7c","remote_addr":"[fd00:1122:3344:101::1]:49912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\", TimeNotSynchronized), (\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\", TimeNotSynchronized), (\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\", TimeNotSynchronized), (\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\", TimeNotSynchronized), (\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\", TimeNotSynchronized)]","latency_us":185298,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:11.581578767Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:477","retry_after":"3.998121158s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ed4ed590-a0d9-4bb2-bfca-f10a930a9a7c\", \"content-length\": \"124\", \"date\": \"Fri, 23 Aug 2024 21:53:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ed4ed590-a0d9-4bb2-bfca-f10a930a9a7c\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bd1e844a-019f-4d2c-af1c-ec9d27991436, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 07617dac-1d06-4704-81b5-9345e9dd66a9, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d738a4fe-561c-4114-8fad-21f02e2dba0e, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ec50006e-a21c-4820-b439-48f6d6b60c97 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cfa27805-1454-49c2-aa13-243fa7ea6adb, kind: Service { id: d738a4fe-561c-4114-8fad-21f02e2dba0e }, name: Name(\"ntp-d738a4fe-561c-4114-8fad-21f02e2dba0e\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 151, 153])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } } } }, OmicronZoneConfig { id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } } } }, OmicronZoneConfig { id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } } } }, OmicronZoneConfig { id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:15.592004081Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:440"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:15.59246203Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:15.592471846Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:15.613821699Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:15.613857625Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:15.633331266Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:15.633432992Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:15.65248178Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:15.652525609Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:15.672980559Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:15.673015669Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:15.693591886Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:15.693627249Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:15.712182452Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:15.71222253Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:15.731143076Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:15.731189071Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:15.749867066Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:15.749913857Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:15.750105345Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:15.750235519Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:15.782619855Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:15.782668447Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:15.782675703Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:15.78268415Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:15.782692823Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:15.782853049Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"ae14b2c4-9131-463a-b959-13d518b27fa0","remote_addr":"[fd00:1122:3344:101::1]:49912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\", TimeNotSynchronized), (\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\", TimeNotSynchronized), (\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\", TimeNotSynchronized), (\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\", TimeNotSynchronized), (\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\", TimeNotSynchronized)]","latency_us":190116,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:15.782993424Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:477","retry_after":"10.694990668s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ae14b2c4-9131-463a-b959-13d518b27fa0\", \"content-length\": \"124\", \"date\": \"Fri, 23 Aug 2024 21:53:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ae14b2c4-9131-463a-b959-13d518b27fa0\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:19.223630526Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:40029"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:19.223697427Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"11bf1259-1df2-4997-a63c-ee9d0a28c942","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:40029","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":44,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:22.137357525Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"991.885065ms"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bd1e844a-019f-4d2c-af1c-ec9d27991436, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 07617dac-1d06-4704-81b5-9345e9dd66a9, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d738a4fe-561c-4114-8fad-21f02e2dba0e, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ec50006e-a21c-4820-b439-48f6d6b60c97 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cfa27805-1454-49c2-aa13-243fa7ea6adb, kind: Service { id: d738a4fe-561c-4114-8fad-21f02e2dba0e }, name: Name(\"ntp-d738a4fe-561c-4114-8fad-21f02e2dba0e\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 151, 153])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } } } }, OmicronZoneConfig { id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } } } }, OmicronZoneConfig { id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } } } }, OmicronZoneConfig { id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:26.508050697Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:440"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:26.508388446Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:26.508398927Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:26.530408405Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:26.53045288Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:26.549366002Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:26.549410637Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:26.567794131Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:26.567831501Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:26.587053845Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:26.587092696Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:26.605705676Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:26.605746673Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:26.624063344Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:26.624099983Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:26.64220061Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:26.642237548Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:26.660871297Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:26.660917254Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:26.661178409Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:26.661316073Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:26.692795438Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:26.692840636Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:26.692850433Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:26.692859134Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:26.692868517Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:26.693008217Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"c3fbb6f8-9988-4f00-8c66-c40e60e7082a","remote_addr":"[fd00:1122:3344:101::1]:49912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\", TimeNotSynchronized), (\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\", TimeNotSynchronized), (\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\", TimeNotSynchronized), (\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\", TimeNotSynchronized), (\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\", TimeNotSynchronized)]","latency_us":184335,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:26.693211995Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:477","retry_after":"10.821984312s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c3fbb6f8-9988-4f00-8c66-c40e60e7082a\", \"content-length\": \"124\", \"date\": \"Fri, 23 Aug 2024 21:53:26 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c3fbb6f8-9988-4f00-8c66-c40e60e7082a\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bd1e844a-019f-4d2c-af1c-ec9d27991436, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 07617dac-1d06-4704-81b5-9345e9dd66a9, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d738a4fe-561c-4114-8fad-21f02e2dba0e, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ec50006e-a21c-4820-b439-48f6d6b60c97 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cfa27805-1454-49c2-aa13-243fa7ea6adb, kind: Service { id: d738a4fe-561c-4114-8fad-21f02e2dba0e }, name: Name(\"ntp-d738a4fe-561c-4114-8fad-21f02e2dba0e\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 151, 153])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } } } }, OmicronZoneConfig { id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } } } }, OmicronZoneConfig { id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } } } }, OmicronZoneConfig { id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:37.54636281Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:440"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:37.546642921Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:37.546653938Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:37.569194694Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:37.569235842Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:37.587988613Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:37.588081996Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:37.60760146Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:37.607629585Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:37.626215556Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:37.626240401Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:37.645228712Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:37.645254624Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:37.663918174Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:37.663946891Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:37.682758946Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:37.682791731Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:37.701014975Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:37.701049363Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:37.701382957Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:37.701510787Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:37.734202245Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:37.734238114Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:37.734247245Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:37.734256461Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:37.734263549Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:37.734367838Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"44e66ef1-a1b3-486f-8561-97b1c876a8c7","remote_addr":"[fd00:1122:3344:101::1]:49912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\", TimeNotSynchronized), (\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\", TimeNotSynchronized), (\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\", TimeNotSynchronized), (\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\", TimeNotSynchronized), (\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\", TimeNotSynchronized)]","latency_us":187413,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:37.734597106Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:477","retry_after":"18.268614743s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"44e66ef1-a1b3-486f-8561-97b1c876a8c7\", \"content-length\": \"124\", \"date\": \"Fri, 23 Aug 2024 21:53:37 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"44e66ef1-a1b3-486f-8561-97b1c876a8c7\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:38.207857767Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"1.789267315s"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:49.337386974Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:47150"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:49.337462884Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"50cc4b52-0f8c-4a57-9111-8e5ad5c2b583","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:47150","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":32,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:55.078018003Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"4.052496193s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bd1e844a-019f-4d2c-af1c-ec9d27991436, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 07617dac-1d06-4704-81b5-9345e9dd66a9, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d738a4fe-561c-4114-8fad-21f02e2dba0e, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ec50006e-a21c-4820-b439-48f6d6b60c97 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cfa27805-1454-49c2-aa13-243fa7ea6adb, kind: Service { id: d738a4fe-561c-4114-8fad-21f02e2dba0e }, name: Name(\"ntp-d738a4fe-561c-4114-8fad-21f02e2dba0e\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 151, 153])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } } } }, OmicronZoneConfig { id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } } } }, OmicronZoneConfig { id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } } } }, OmicronZoneConfig { id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:56.054009223Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:440"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:56.054365424Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:56.054376149Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:56.076275756Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:56.076347972Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:56.096176383Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:56.096226484Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:56.114909024Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:56.114946947Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:56.134076598Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:56.134121476Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:56.151908182Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:56.151952227Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:56.171078507Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:56.171122551Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:56.189279854Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:56.189338543Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:56.208045501Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:56.208096707Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:56.208363046Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:56.208477312Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:56.241352317Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:56.24138934Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:56.241397787Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:56.241406112Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:56.241412441Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:53:56.241507204Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"527405cb-c938-47a6-836b-99c481eb9742","remote_addr":"[fd00:1122:3344:101::1]:49912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\", TimeNotSynchronized), (\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\", TimeNotSynchronized), (\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\", TimeNotSynchronized), (\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\", TimeNotSynchronized), (\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\", TimeNotSynchronized)]","latency_us":186864,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:53:56.241639703Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:477","retry_after":"10.052432848s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"527405cb-c938-47a6-836b-99c481eb9742\", \"content-length\": \"124\", \"date\": \"Fri, 23 Aug 2024 21:53:56 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"527405cb-c938-47a6-836b-99c481eb9742\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bd1e844a-019f-4d2c-af1c-ec9d27991436, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 07617dac-1d06-4704-81b5-9345e9dd66a9, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d738a4fe-561c-4114-8fad-21f02e2dba0e, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ec50006e-a21c-4820-b439-48f6d6b60c97 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cfa27805-1454-49c2-aa13-243fa7ea6adb, kind: Service { id: d738a4fe-561c-4114-8fad-21f02e2dba0e }, name: Name(\"ntp-d738a4fe-561c-4114-8fad-21f02e2dba0e\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 151, 153])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } } } }, OmicronZoneConfig { id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } } } }, OmicronZoneConfig { id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } } } }, OmicronZoneConfig { id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:06.322367657Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:440"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:06.322672658Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:06.322683524Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:06.344181421Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:06.344225147Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:06.362819669Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:06.362868242Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:06.380506459Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:06.380543734Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:06.401762964Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:06.401792251Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:06.421163999Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:06.421195068Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:06.439503279Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:06.439533907Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:06.458713831Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:06.458744947Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:06.477077239Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:06.477107765Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:06.477351313Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:06.477537421Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:54:06.510660274Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:54:06.510708041Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:54:06.510716937Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:54:06.510723482Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:54:06.510729923Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:06.510788892Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"ecaf14cd-c5a2-4769-bc38-795ea93538bf","remote_addr":"[fd00:1122:3344:101::1]:49912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\", TimeNotSynchronized), (\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\", TimeNotSynchronized), (\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\", TimeNotSynchronized), (\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\", TimeNotSynchronized), (\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\", TimeNotSynchronized)]","latency_us":187762,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:54:06.510913179Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:477","retry_after":"16.323152492s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ecaf14cd-c5a2-4769-bc38-795ea93538bf\", \"content-length\": \"124\", \"date\": \"Fri, 23 Aug 2024 21:54:06 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ecaf14cd-c5a2-4769-bc38-795ea93538bf\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:54:14.221113631Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"11.17373946s"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:19.451531821Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36101"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:19.451621992Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"7bcd86b3-09a0-4d45-96f8-a43a574a281d","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36101","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":52,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bd1e844a-019f-4d2c-af1c-ec9d27991436, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 07617dac-1d06-4704-81b5-9345e9dd66a9, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d738a4fe-561c-4114-8fad-21f02e2dba0e, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ec50006e-a21c-4820-b439-48f6d6b60c97 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cfa27805-1454-49c2-aa13-243fa7ea6adb, kind: Service { id: d738a4fe-561c-4114-8fad-21f02e2dba0e }, name: Name(\"ntp-d738a4fe-561c-4114-8fad-21f02e2dba0e\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 151, 153])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } } } }, OmicronZoneConfig { id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } } } }, OmicronZoneConfig { id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } } } }, OmicronZoneConfig { id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:22.879401716Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:440"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:22.879747246Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:22.879758693Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:22.901520519Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:22.901591912Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:22.921455578Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:22.921495207Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:22.940180413Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:22.940217398Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:22.959711457Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:22.959743445Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:22.97775497Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:22.977790755Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:22.997406136Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:22.997458374Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:23.016834754Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:23.016876633Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:23.03593704Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:23.03597438Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:23.036189184Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:23.036326521Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:54:23.068300708Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:54:23.068347592Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:54:23.068356873Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:54:23.068364449Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:54:23.068373496Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:23.068553771Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"3d1a9170-1420-478b-a0ca-a870d2400aa8","remote_addr":"[fd00:1122:3344:101::1]:49912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\", TimeNotSynchronized), (\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\", TimeNotSynchronized), (\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\", TimeNotSynchronized), (\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\", TimeNotSynchronized), (\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\", TimeNotSynchronized)]","latency_us":188450,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:54:23.068713599Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:477","retry_after":"25.630528577s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3d1a9170-1420-478b-a0ca-a870d2400aa8\", \"content-length\": \"124\", \"date\": \"Fri, 23 Aug 2024 21:54:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3d1a9170-1420-478b-a0ca-a870d2400aa8\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:54:40.502499665Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"19.482206901s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bd1e844a-019f-4d2c-af1c-ec9d27991436, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 07617dac-1d06-4704-81b5-9345e9dd66a9, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d738a4fe-561c-4114-8fad-21f02e2dba0e, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ec50006e-a21c-4820-b439-48f6d6b60c97 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cfa27805-1454-49c2-aa13-243fa7ea6adb, kind: Service { id: d738a4fe-561c-4114-8fad-21f02e2dba0e }, name: Name(\"ntp-d738a4fe-561c-4114-8fad-21f02e2dba0e\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 151, 153])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } } } }, OmicronZoneConfig { id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } } } }, OmicronZoneConfig { id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } } } }, OmicronZoneConfig { id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:48.770443948Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:440"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:48.770748379Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:48.77076216Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:48.793353031Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:48.79340482Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:48.813336556Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:48.813371975Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:48.83219182Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:48.832223161Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:48.850703079Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:48.850731692Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:48.870002228Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:48.870031695Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:48.88826891Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:48.888304085Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:48.906366787Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:48.906392409Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:48.92569455Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:48.925747051Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:48.926005731Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:48.926122864Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:54:48.958329927Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:54:48.958363809Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:54:48.958371872Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:54:48.958377994Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:54:48.958384369Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:48.958495127Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"77f93f36-ee8c-44f9-aec9-54ba57e05d20","remote_addr":"[fd00:1122:3344:101::1]:49912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\", TimeNotSynchronized), (\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\", TimeNotSynchronized), (\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\", TimeNotSynchronized), (\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\", TimeNotSynchronized), (\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\", TimeNotSynchronized)]","latency_us":187450,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:54:48.958725812Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:477","retry_after":"20.720843812s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"77f93f36-ee8c-44f9-aec9-54ba57e05d20\", \"content-length\": \"124\", \"date\": \"Fri, 23 Aug 2024 21:54:48 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"77f93f36-ee8c-44f9-aec9-54ba57e05d20\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:49.565484913Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:49223"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:54:49.56555534Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"a6b3d9d5-6b98-4975-b1bc-9278de989517","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:49223","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":40,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bd1e844a-019f-4d2c-af1c-ec9d27991436, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 07617dac-1d06-4704-81b5-9345e9dd66a9, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d738a4fe-561c-4114-8fad-21f02e2dba0e, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ec50006e-a21c-4820-b439-48f6d6b60c97 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cfa27805-1454-49c2-aa13-243fa7ea6adb, kind: Service { id: d738a4fe-561c-4114-8fad-21f02e2dba0e }, name: Name(\"ntp-d738a4fe-561c-4114-8fad-21f02e2dba0e\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 151, 153])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } } } }, OmicronZoneConfig { id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } } } }, OmicronZoneConfig { id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } } } }, OmicronZoneConfig { id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:55:09.73725352Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:440"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:55:09.737571901Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:55:09.737583198Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:55:09.759664832Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:55:09.759721074Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:55:09.779459141Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:55:09.779487997Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:55:09.798141978Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:55:09.798239889Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:55:09.815998693Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:55:09.816033296Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:55:09.834767258Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:55:09.834795216Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:55:09.853097486Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:55:09.853140574Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:55:09.872127998Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:55:09.872165432Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:55:09.890501008Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:55:09.890561787Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:55:09.890804989Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:55:09.890946984Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:55:09.922679028Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:55:09.922713237Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:55:09.922722659Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:55:09.922731651Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:55:09.922740332Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3249","zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:55:09.922834703Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"5e47ec52-beaf-4992-b6ca-e41b9e75a690","remote_addr":"[fd00:1122:3344:101::1]:49912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\", TimeNotSynchronized), (\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\", TimeNotSynchronized), (\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\", TimeNotSynchronized), (\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\", TimeNotSynchronized), (\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\", TimeNotSynchronized)]","latency_us":184954,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:55:09.923009268Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:477","retry_after":"51.161663737s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5e47ec52-beaf-4992-b6ca-e41b9e75a690\", \"content-length\": \"124\", \"date\": \"Fri, 23 Aug 2024 21:55:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5e47ec52-beaf-4992-b6ca-e41b9e75a690\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:55:15.115443856Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"33.020207597s"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:55:19.680207818Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:57000"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:55:19.68026105Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"29f57eda-6e11-43e3-ada7-974ffb67f0bf","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:57000","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":30,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:55:49.754257814Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56673"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:55:49.754299655Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"369556e4-9102-4424-90a7-226f17b4b552","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56673","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":33,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bd1e844a-019f-4d2c-af1c-ec9d27991436, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 07617dac-1d06-4704-81b5-9345e9dd66a9, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d738a4fe-561c-4114-8fad-21f02e2dba0e, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ec50006e-a21c-4820-b439-48f6d6b60c97 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cfa27805-1454-49c2-aa13-243fa7ea6adb, kind: Service { id: d738a4fe-561c-4114-8fad-21f02e2dba0e }, name: Name(\"ntp-d738a4fe-561c-4114-8fad-21f02e2dba0e\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 151, 153])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } } } }, OmicronZoneConfig { id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } } } }, OmicronZoneConfig { id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } } } }, OmicronZoneConfig { id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:01.153989226Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:440"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:01.154184789Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:01.15420059Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:01.177199249Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:01.17724541Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:01.196814362Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:01.196848402Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:01.213006126Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:01.213040006Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:01.231455258Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:01.231494378Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:01.250107494Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:01.250142764Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:01.268261391Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:01.268290132Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:01.287313294Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:01.287346334Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:01.306948546Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:01.306989717Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:01.307222411Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:01.307324553Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:01.393292869Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:01.393345669Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:01.393354859Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:01.39336115Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:01.39336992Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:01.48846568Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:01.492497298Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:01.496574047Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:01.501000273Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:01.50560552Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:01.510362661Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:01.515147852Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:01.519566427Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:01.52388259Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:01.528520089Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:03.227023773Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"95.85143174s"} {"msg":"Profile for oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:04.450235337Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:04.450359409Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:04.48113355Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:04.481269733Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:04.554511533Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:04.554632505Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\") failed: Property not found. retry in 47.944608ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:04.604184704Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\") failed: Property not found. retry in 41.041384ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:04.625716509Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:04.626013524Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:04.626466381Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:04.632156848Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:04.632272109Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\") failed: Property not found. retry in 94.72605ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:04.662238857Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\") failed: Property not found. retry in 68.891223ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:04.695807925Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\") failed: Property not found. retry in 208.579833ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:04.900219447Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\") failed: Property not found. retry in 46.248382ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:04.967464736Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\") failed: Property not found. retry in 289.950401ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:04.990090719Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\") failed: Property not found. retry in 97.290575ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:05.077327156Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\") failed: Property not found. retry in 368.550958ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:05.181014912Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\") failed: Property not found. retry in 254.816185ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:05.202310553Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\") failed: Property not found. retry in 66.803349ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:05.232202659Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\") failed: Property not found. retry in 70.099362ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:05.701226112Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\") failed: Property not found. retry in 466.179813ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:05.74422629Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\") failed: Property not found. retry in 252.450263ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:05.975968264Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\") failed: Property not found. retry in 703.485401ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:06.067516095Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\") failed: Property not found. retry in 137.942421ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:06.150005492Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\") failed: Property not found. retry in 135.007436ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:06.201601176Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\") failed: Property not found. retry in 1.12953637s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:06.300226545Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\") failed: Property not found. retry in 1.099106132s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:06.349916077Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\") failed: Property not found. retry in 141.785154ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:06.390370522Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\") failed: Property not found. retry in 215.933822ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:06.475145078Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\") failed: Property not found. retry in 282.277102ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:06.672674073Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\") failed: Property not found. retry in 574.820327ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:06.741230834Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\") failed: Property not found. retry in 512.572868ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:06.836702001Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\") failed: Property not found. retry in 535.376473ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:06.984112157Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\") failed: Property not found. retry in 1.131177717s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:07.327616894Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\") failed: Property not found. retry in 1.356130097s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:07.359788529Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\") failed: Property not found. retry in 1.030984555s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:07.44192061Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\") failed: Property not found. retry in 1.132097229s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:07.460784889Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\") failed: Property not found. retry in 1.328973996s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:07.529401571Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\") failed: Property not found. retry in 634.03747ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:08.468674828Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\") failed: Property not found. retry in 1.488064218s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:08.4841476Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\") failed: Property not found. retry in 1.262911269s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:08.601346715Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\") failed: Property not found. retry in 979.240081ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:08.725723251Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\") failed: Property not found. retry in 808.205963ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:08.869455295Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\") failed: Property not found. retry in 663.341246ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:09.11419373Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\") failed: Property not found. retry in 860.085479ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:09.687140103Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\") failed: Property not found. retry in 688.048908ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:09.714250952Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\") failed: Property not found. retry in 605.651348ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:09.786375603Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\") failed: Property not found. retry in 1.245932285s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:09.873063601Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\") failed: Property not found. retry in 1.379421864s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:09.981577549Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\") failed: Property not found. retry in 540.05765ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:10.403560855Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\") failed: Property not found. retry in 1.116438506s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:10.412338074Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\") failed: Property not found. retry in 1.3272214s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:10.557317979Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\") failed: Property not found. retry in 1.072808117s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:10.952445871Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\") failed: Property not found. retry in 751.287348ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:11.13073448Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\") failed: Property not found. retry in 656.992722ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:11.370738544Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\") failed: Property not found. retry in 1.006954262s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:11.538127839Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\") failed: Property not found. retry in 1.268649132s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:11.892107823Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\") failed: Property not found. retry in 1.3775573s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:11.900670658Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\") failed: Property not found. retry in 1.062440756s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:12.035158866Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\") failed: Property not found. retry in 591.472402ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:12.043272993Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\") failed: Property not found. retry in 1.431071748s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:12.555531087Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\") failed: Property not found. retry in 970.93436ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:12.643476997Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\") failed: Property not found. retry in 963.612637ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:13.107501053Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\") failed: Property not found. retry in 799.026933ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:13.169951068Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\") failed: Property not found. retry in 1.413538194s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:13.287773454Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\") failed: Property not found. retry in 867.325799ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:13.624116902Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\") failed: Property not found. retry in 1.244092491s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:13.978759193Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\") failed: Property not found. retry in 725.756436ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:13.995522282Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\") failed: Property not found. retry in 1.167890024s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:14.081874655Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\") failed: Property not found. retry in 1.029512772s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:14.501824536Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\") failed: Property not found. retry in 1.179248907s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:14.711360465Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\") failed: Property not found. retry in 757.799437ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:14.730900635Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\") failed: Property not found. retry in 1.13669134s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:15.231523053Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\") failed: Property not found. retry in 1.18995371s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:15.259198801Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\") failed: Property not found. retry in 697.797982ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:15.500003689Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\") failed: Property not found. retry in 1.169732026s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:15.5395879Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\") failed: Property not found. retry in 1.393548775s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:15.897928248Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\") failed: Property not found. retry in 1.379984691s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:16.206524324Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\") failed: Property not found. retry in 830.063408ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:16.377206184Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\") failed: Property not found. retry in 1.115078673s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:16.458045923Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\") failed: Property not found. retry in 1.109478709s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:16.71769765Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\") failed: Property not found. retry in 1.459894894s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:17.216370385Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\") failed: Property not found. retry in 1.042558891s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:17.301439125Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\") failed: Property not found. retry in 1.274443555s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:17.581761863Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\") failed: Property not found. retry in 807.417029ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:17.59517746Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\") failed: Property not found. retry in 1.097634164s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:17.836353564Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\") failed: Property not found. retry in 1.415237383s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:18.353644884Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\") failed: Property not found. retry in 1.15226554s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:18.412157985Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\") failed: Property not found. retry in 1.462719987s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:18.68524675Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\") failed: Property not found. retry in 698.138862ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:18.86537651Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\") failed: Property not found. retry in 1.17200926s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:18.94384857Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\") failed: Property not found. retry in 987.208553ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:19.572374743Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\") failed: Property not found. retry in 685.21146ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:19.579819589Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\") failed: Property not found. retry in 1.038371171s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:19.779951138Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:19.788987787Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42994"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:19.789241846Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"1f150717-74cd-4720-86ca-acead49ae127","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42994","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":99,"response_code":"200"} {"msg":"wait for service Some(\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\") failed: Property not found. retry in 1.254561484s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:20.127083877Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\") failed: Property not found. retry in 517.112871ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:20.158677502Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\") failed: Property not found. retry in 1.488846355s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:20.278664534Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\") failed: Property not found. retry in 1.046634962s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:20.569909556Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\") failed: Property not found. retry in 1.001851316s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:20.685677937Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\") failed: Property not found. retry in 782.810401ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:20.829160777Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\") failed: Property not found. retry in 1.275512778s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:21.391022722Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\") failed: Property not found. retry in 1.289673227s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:21.622315369Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\") failed: Property not found. retry in 1.030839282s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:21.633697111Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\") failed: Property not found. retry in 504.530795ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:21.697229647Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\") failed: Property not found. retry in 900.87591ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:21.776530261Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\") failed: Property not found. retry in 829.501627ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:22.211982615Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\") failed: Property not found. retry in 1.043004627s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:22.674130371Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\") failed: Property not found. retry in 1.240633468s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:22.681937803Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\") failed: Property not found. retry in 669.4798ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:22.689619973Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\") failed: Property not found. retry in 925.317333ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:22.922446126Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\") failed: Property not found. retry in 1.479138277s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:23.050944622Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\") failed: Property not found. retry in 701.76019ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:23.368202215Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\") failed: Property not found. retry in 539.556294ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:23.726631285Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\") failed: Property not found. retry in 751.45629ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:23.857647764Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\") failed: Property not found. retry in 515.870818ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:23.932100974Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855\") failed: Property not found. retry in 778.213932ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:24.08003341Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\") failed: Property not found. retry in 629.839682ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:24.275907497Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657\") failed: Property not found. retry in 503.58354ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:24.459490425Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb\") failed: Property not found. retry in 1.004601099s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:24.540995066Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8\") failed: Property not found. retry in 1.453245078s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:24.619907892Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd\") failed: Property not found. retry in 631.955677ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:24.970448349Z","hostname":"buskin","pid":1562,"zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:26.097573996Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_cockroachdb_5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:26.097641007Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_cockroachdb_32a152fc-12d5-4442-9ec7-44167f70e8eb"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:26.097649698Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_cockroachdb_3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:26.097655768Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_cockroachdb_a32a4cfc-b6ee-416e-8738-c10a4146b4f8"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:26.097662148Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_cockroachdb_5749b096-fa59-4034-bf5c-f4e555b65bfd"} {"msg":"Writing ledger to /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:26.097666688Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:26.098079845Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:26.09841054Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"a4a5c932-0505-48aa-8dee-560a2c1e6dfd","remote_addr":"[fd00:1122:3344:101::1]:49912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":24943214,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:26.133614887Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:57137"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::4]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:26.164298846Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3681"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:37.6817054Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3699"} {"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:41.695723507Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3722"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:41.695845419Z","hostname":"buskin","pid":1562,"uri":"/cockroachdb","method":"POST","req_id":"b380356e-5e95-4513-9de6-7581e53fc039","remote_addr":"[fd00:1122:3344:101::1]:57137","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":15561941,"response_code":"204"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bd1e844a-019f-4d2c-af1c-ec9d27991436, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 07617dac-1d06-4704-81b5-9345e9dd66a9, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: d738a4fe-561c-4114-8fad-21f02e2dba0e, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ec50006e-a21c-4820-b439-48f6d6b60c97 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: cfa27805-1454-49c2-aa13-243fa7ea6adb, kind: Service { id: d738a4fe-561c-4114-8fad-21f02e2dba0e }, name: Name(\"ntp-d738a4fe-561c-4114-8fad-21f02e2dba0e\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 151, 153])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } } } }, OmicronZoneConfig { id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } } } }, OmicronZoneConfig { id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } } } }, OmicronZoneConfig { id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1db57e91-9837-462c-9f2c-1b4fca780b7e, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2a9b2cf7-2fb3-4597-871e-44332b4d82f2, kind: Service { id: 1db57e91-9837-462c-9f2c-1b4fca780b7e }, name: Name(\"external-dns-1db57e91-9837-462c-9f2c-1b4fca780b7e\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 24])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 5d1239c9-696e-4616-a2a6-82968e91a2e1, kind: Service { id: b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac }, name: Name(\"external-dns-b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 184])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 4e26237f-8cdc-4da1-b44a-9b00d20bd873, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 00000426-1f68-402a-a60e-4fa2f73c115f, kind: Service { id: 4e26237f-8cdc-4da1-b44a-9b00d20bd873 }, name: Name(\"nexus-4e26237f-8cdc-4da1-b44a-9b00d20bd873\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 190, 168])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2c94012a-643d-4f61-8185-c8786e00e320, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b2284d48-b022-4af4-a2f6-3a691122eb62, kind: Service { id: 2c94012a-643d-4f61-8185-c8786e00e320 }, name: Name(\"nexus-2c94012a-643d-4f61-8185-c8786e00e320\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 253, 42])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f9a2f81c-00cf-49a5-a96e-1c4dfeba1728, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5cf061ea-987b-4986-8378-702c868c90c5, kind: Service { id: f9a2f81c-00cf-49a5-a96e-1c4dfeba1728 }, name: Name(\"nexus-f9a2f81c-00cf-49a5-a96e-1c4dfeba1728\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 196, 138])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba8b1177-03a3-49b6-a6ee-b364652c50c9, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e1a016b1-16ad-48d4-a26f-2e916f51bc80 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 8471ffba-0f68-4241-be3b-68581f848fb7, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } } } }, OmicronZoneConfig { id: d9421cb5-f5bb-43e4-a2f0-337cff4f1100, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 8343e6aa-08fb-4b4a-bf94-a818d66d575d, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 7ac03450-9a27-4ec4-9669-9eb798da4419 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1bb815f5-1d70-43b2-a7b7-1bb132d87fd6, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: dad70eae-b83f-4c3a-a32e-2a6a8946803c (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 92744278-0a6d-41bc-a502-b694867741c6, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3dafd3b1-1230-4db3-9961-2a4d53cc3a37, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External } } } }, OmicronZoneConfig { id: 66dd6deb-9c39-47ef-9f1c-98fea2467257, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External } } } }, OmicronZoneConfig { id: 9c2bbea7-8b8c-4d83-9618-26c0848936eb, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External } } } }, OmicronZoneConfig { id: 914d3516-8bfd-4006-87d8-91481730f9c7, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2bda41aa-8135-413a-853b-44e7fc746744, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: dad70eae-b83f-4c3a-a32e-2a6a8946803c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dad70eae-b83f-4c3a-a32e-2a6a8946803c (zpool), kind: External } } } }, OmicronZoneConfig { id: 02433f6a-f8a9-4f05-af1b-be59dfd00a41, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: ec50006e-a21c-4820-b439-48f6d6b60c97 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ec50006e-a21c-4820-b439-48f6d6b60c97 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83653856-00c4-44fe-b4df-ad99e78af473, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: e1a016b1-16ad-48d4-a26f-2e916f51bc80 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e1a016b1-16ad-48d4-a26f-2e916f51bc80 (zpool), kind: External } } } }, OmicronZoneConfig { id: 581275cf-e6a7-4ef5-ad1e-36b1379b2a18, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 7ac03450-9a27-4ec4-9669-9eb798da4419 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ac03450-9a27-4ec4-9669-9eb798da4419 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:41.735916737Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:440"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:41.736104691Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:55451"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:41.736754711Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:41.736769512Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: bd1e844a-019f-4d2c-af1c-ec9d27991436, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:41.767721546Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:41.767783157Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 60c8c02e-6e6f-4e1a-a162-ca6470555313, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:41.79157478Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:41.791616591Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 07617dac-1d06-4704-81b5-9345e9dd66a9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:41.813001513Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:41.813040433Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5c4d7b3b-b9a4-4c5d-9fa8-057fdf1ca657, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:41.834066649Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:41.83410574Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 32a152fc-12d5-4442-9ec7-44167f70e8eb, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:41.854540016Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:41.854581387Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: a32a4cfc-b6ee-416e-8738-c10a4146b4f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:41.876829194Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:41.876884744Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 5749b096-fa59-4034-bf5c-f4e555b65bfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:41.897343701Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:41.897388452Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 3a0f6ecf-1b05-4bf6-8a0a-f7e485f52855, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1db57e91-9837-462c-9f2c-1b4fca780b7e, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:41.919572738Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:41.919618688Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 1db57e91-9837-462c-9f2c-1b4fca780b7e, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:41.998568225Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:41.998623826Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8471ffba-0f68-4241-be3b-68581f848fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:42.08564822Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:42.085703251Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 8471ffba-0f68-4241-be3b-68581f848fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 92744278-0a6d-41bc-a502-b694867741c6, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:42.162344549Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:42.16239656Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 92744278-0a6d-41bc-a502-b694867741c6, dataset_name: DatasetName { pool_name: ZpoolName { id: c4828395-89b1-4653-93a7-55b242871c48 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3dafd3b1-1230-4db3-9961-2a4d53cc3a37, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:42.241698582Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:42.241750733Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 3dafd3b1-1230-4db3-9961-2a4d53cc3a37, dataset_name: DatasetName { pool_name: ZpoolName { id: c4fe4e01-40de-4c9e-857a-e6870bd84153 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 66dd6deb-9c39-47ef-9f1c-98fea2467257, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:42.316662272Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:42.316714953Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 66dd6deb-9c39-47ef-9f1c-98fea2467257, dataset_name: DatasetName { pool_name: ZpoolName { id: 0bb3d35c-cbec-445b-9b3a-58337934f8de (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9c2bbea7-8b8c-4d83-9618-26c0848936eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:42.396726208Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:42.396774679Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 9c2bbea7-8b8c-4d83-9618-26c0848936eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b92f88-0c07-4732-953e-9cac63715495 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 914d3516-8bfd-4006-87d8-91481730f9c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:42.471938692Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:42.471999173Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 914d3516-8bfd-4006-87d8-91481730f9c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 44ff68f6-909f-4df8-8e90-d68ca84323b3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2bda41aa-8135-413a-853b-44e7fc746744, dataset_name: DatasetName { pool_name: ZpoolName { id: dad70eae-b83f-4c3a-a32e-2a6a8946803c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:42.55040656Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:42.550457571Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 2bda41aa-8135-413a-853b-44e7fc746744, dataset_name: DatasetName { pool_name: ZpoolName { id: dad70eae-b83f-4c3a-a32e-2a6a8946803c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 02433f6a-f8a9-4f05-af1b-be59dfd00a41, dataset_name: DatasetName { pool_name: ZpoolName { id: ec50006e-a21c-4820-b439-48f6d6b60c97 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:42.61595274Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:42.616004051Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 02433f6a-f8a9-4f05-af1b-be59dfd00a41, dataset_name: DatasetName { pool_name: ZpoolName { id: ec50006e-a21c-4820-b439-48f6d6b60c97 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83653856-00c4-44fe-b4df-ad99e78af473, dataset_name: DatasetName { pool_name: ZpoolName { id: e1a016b1-16ad-48d4-a26f-2e916f51bc80 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:42.67970136Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:42.679747631Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 83653856-00c4-44fe-b4df-ad99e78af473, dataset_name: DatasetName { pool_name: ZpoolName { id: e1a016b1-16ad-48d4-a26f-2e916f51bc80 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 581275cf-e6a7-4ef5-ad1e-36b1379b2a18, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ac03450-9a27-4ec4-9669-9eb798da4419 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:42.746619553Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:42.746671014Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:773","request":"NewFilesystemRequest { dataset_id: 581275cf-e6a7-4ef5-ad1e-36b1379b2a18, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ac03450-9a27-4ec4-9669-9eb798da4419 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:42.812451248Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:42.812524749Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:42.812786524Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:42.813060618Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:42.940417945Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:260"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:42.940610058Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:42.957758209Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:260"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:42.957921472Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:42.957942792Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:42.990354031Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:260"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:42.990579705Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:42.990623236Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:42.990647956Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:42.992245283Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:260"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:43.146941603Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:260"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:43.147156656Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:43.147223737Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:43.207332115Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:43.207384496Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:43.207398427Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:43.207402197Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:43.207406357Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:43.207409607Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Configuring new Omicron zone: oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:43.207502448Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:43.214211642Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:43.22120853Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:43.228008265Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:43.233728382Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:43.239800595Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:43.413529587Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:43.418949659Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:43.424469362Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:43.428995949Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:43.432822174Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:43.437430492Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:43.44325741Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:43.448681522Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:43.454668304Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:43.460752887Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:43.466565425Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_clickhouse_8471ffba-0f68-4241-be3b-68581f848fb7","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:43.473651265Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:43.480850277Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:43.487168954Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:43.541200059Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"} {"msg":"Installing Omicron zone: oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:43.542138675Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:43.550664549Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:43.559195424Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:43.568032983Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:43.577050806Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_clickhouse_8471ffba-0f68-4241-be3b-68581f848fb7","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:43.585780834Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:43.595942996Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:43.606244081Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:43.693636661Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:43.882180194Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:43.943840728Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.004845451Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.008657136Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:302"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.008678726Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.126977189Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:424","port":"Port { inner: PortInner(PortData { name: \"opte1\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 231, 24]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.12701166Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1235","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.128874212Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:302"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.128896402Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.186133811Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:424","port":"Port { inner: PortInner(PortData { name: \"opte2\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 196, 138]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.186248393Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1235","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.188287838Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:302"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.188307578Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.247139504Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:424","port":"Port { inner: PortInner(PortData { name: \"opte3\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 163, 184]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.247171435Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1235","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.248608709Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:302"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.248628669Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.300563799Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:424","port":"Port { inner: PortInner(PortData { name: \"opte4\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 253, 42]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.300594439Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1235","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.302889408Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:302"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.302902988Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.358833185Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:424","port":"Port { inner: PortInner(PortData { name: \"opte5\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 190, 168]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.358858076Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1235","zone_type":"nexus"} {"msg":"no nat entry found for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.358877046Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.358893816Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.358908407Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.358914327Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.359172131Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.359188472Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.359240452Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.359252633Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.384732394Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.386321841Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"creation of nat entry successful for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.431297942Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"creation of nat entry successful for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.434926304Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.435092217Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"creation of nat entry successful for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.438320552Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.438485454Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.441712859Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"no nat entry found for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.441735279Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.441743239Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.58763875Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.591416284Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Configuring new Omicron zone: oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.591611428Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.604925213Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_external_dns_b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.615240478Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_nexus_2c94012a-643d-4f61-8185-c8786e00e320","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.627494595Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.685395536Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.694781115Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.714923116Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_2c94012a-643d-4f61-8185-c8786e00e320","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.739023494Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.756636802Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:44.810294931Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:46.821943988Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:46.822145931Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:46.849728648Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:46.849913801Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:46.965495029Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:46.965702102Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:46.979646648Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:46.979884232Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:46.988792393Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:46.989599877Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:47.000767156Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:47.00099173Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:47.047258213Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:47.047508318Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:47.11081984Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:47.110933012Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:47.121338238Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:47.121665843Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:47.132554018Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:47.132738321Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:47.149365363Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:47.14980588Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6\") failed: Property not found. retry in 44.432858ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:48.737139511Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37\") failed: Property not found. retry in 44.813711ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:54.395322901Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7\") failed: Property not found. retry in 52.169421ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:55.17827574Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6\") failed: Property not found. retry in 74.899339ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:55.197002677Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6\") failed: Property not found. retry in 141.344141ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:55.349762164Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9\") failed: Property not found. retry in 58.447375ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:55.449035115Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37\") failed: Property not found. retry in 89.49052ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:55.544151286Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100\") failed: Property not found. retry in 68.350564ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:55.702498486Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257\") failed: Property not found. retry in 42.853844ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:55.74633711Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb\") failed: Property not found. retry in 29.711969ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:55.777283334Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:55.819293986Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18\") failed: Property not found. retry in 61.168395ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:55.83492999Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d\") failed: Property not found. retry in 62.524802ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:55.86031227Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:55.922689366Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41\") failed: Property not found. retry in 67.086924ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:55.952354939Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7\") failed: Property not found. retry in 131.64085ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:55.978185816Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6\") failed: Property not found. retry in 143.089513ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:56.002768113Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6\") failed: Property not found. retry in 165.003488ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:56.015164133Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9\") failed: Property not found. retry in 145.703664ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:56.028508269Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37\") failed: Property not found. retry in 134.524242ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:56.044258205Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100\") failed: Property not found. retry in 67.033322ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:56.07110748Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257\") failed: Property not found. retry in 130.355428ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:56.09767192Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb\") failed: Property not found. retry in 139.215243ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:56.11184669Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:56.111953402Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18\") failed: Property not found. retry in 128.53612ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:56.156782511Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d\") failed: Property not found. retry in 142.428939ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:56.179041942Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:56.179159469Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41\") failed: Property not found. retry in 56.79429ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:56.228254261Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7\") failed: Property not found. retry in 293.660846ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:56.244917283Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6\") failed: Property not found. retry in 224.004466ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:56.257434165Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6\") failed: Property not found. retry in 458.198024ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:56.270450796Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9\") failed: Property not found. retry in 210.532892ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:56.283014648Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37\") failed: Property not found. retry in 478.4974ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:56.295054802Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100\") failed: Property not found. retry in 267.678474ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:56.306108439Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257\") failed: Property not found. retry in 122.205956ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:56.31791499Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb\") failed: Property not found. retry in 291.713408ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:56.33922536Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18\") failed: Property not found. retry in 161.550424ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:56.352363273Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d\") failed: Property not found. retry in 298.107409ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:56.363509471Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41\") failed: Property not found. retry in 146.789262ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:56.374592549Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744\") failed: Property not found. retry in 73.542299ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:56.420263923Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473\") failed: Property not found. retry in 33.8274ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:56.431573674Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257\") failed: Property not found. retry in 295.947691ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:56.454261659Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473\") failed: Property not found. retry in 143.012309ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:56.476509845Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6\") failed: Property not found. retry in 261.668668ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:56.4951025Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9\") failed: Property not found. retry in 459.938941ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:56.510975229Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744\") failed: Property not found. retry in 122.332515ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:56.524121922Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18\") failed: Property not found. retry in 384.113412ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:56.538971683Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41\") failed: Property not found. retry in 558.976963ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:56.552837998Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7\") failed: Property not found. retry in 377.384523ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:56.567509696Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100\") failed: Property not found. retry in 508.246811ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:56.586488548Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473\") failed: Property not found. retry in 261.154502ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:56.633755108Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb\") failed: Property not found. retry in 579.539565ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:56.648284754Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744\") failed: Property not found. retry in 260.542441ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:56.661288645Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d\") failed: Property not found. retry in 251.326515ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:56.674915805Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6\") failed: Property not found. retry in 843.213939ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:56.744416622Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257\") failed: Property not found. retry in 1.179074894s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:56.765167524Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6\") failed: Property not found. retry in 654.722173ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:56.777786267Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37\") failed: Property not found. retry in 886.227842ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:56.789291452Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473\") failed: Property not found. retry in 290.057933ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:56.911700035Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:56.913625758Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:56.914645035Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744\") failed: Property not found. retry in 401.897111ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:56.938693532Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18\") failed: Property not found. retry in 767.010846ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:56.952082759Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d\") failed: Property not found. retry in 1.071650793s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:56.966024245Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7\") failed: Property not found. retry in 540.074432ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:56.980108794Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9\") failed: Property not found. retry in 561.637428ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:56.994479487Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100\") failed: Property not found. retry in 1.02632558s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:57.106384912Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41\") failed: Property not found. retry in 912.03516ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:57.129004445Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e\") failed: Property not found. retry in 40.808154ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:57.174387754Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473\") failed: Property not found. retry in 1.144287566s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:57.216546518Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e\") failed: Property not found. retry in 94.970545ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:57.228266416Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb\") failed: Property not found. retry in 1.124099439s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:57.240763188Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e\") failed: Property not found. retry in 237.371496ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:57.335808196Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744\") failed: Property not found. retry in 1.039490117s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:57.353256093Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6\") failed: Property not found. retry in 931.218283ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:57.446842268Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7\") failed: Property not found. retry in 528.085985ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:57.533872502Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9\") failed: Property not found. retry in 925.199754ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:57.569272651Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e\") failed: Property not found. retry in 362.778061ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:57.586805388Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6\") failed: Property not found. retry in 550.573982ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:57.602672757Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37\") failed: Property not found. retry in 1.124971399s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:57.68968109Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18\") failed: Property not found. retry in 1.442245557s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:57.732000917Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257\") failed: Property not found. retry in 1.129348502s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:57.957818221Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e\") failed: Property not found. retry in 694.791646ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:57.970618398Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:57.993884572Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:57.993930503Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:57.994214347Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:56:58.001931008Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d\") failed: Property not found. retry in 677.617475ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:58.051265174Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41\") failed: Property not found. retry in 1.304425663s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:58.062070997Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7\") failed: Property not found. retry in 943.229194ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:58.075301181Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100\") failed: Property not found. retry in 1.436546632s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:58.145201475Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6\") failed: Property not found. retry in 1.228841576s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:58.166296922Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873\") failed: Property not found. retry in 60.887988ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:58.214279274Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728\") failed: Property not found. retry in 55.439424ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:58.224197152Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873\") failed: Property not found. retry in 105.867429ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:58.286712521Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728\") failed: Property not found. retry in 106.791528ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:58.295901006Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473\") failed: Property not found. retry in 513.766005ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:58.373389899Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb\") failed: Property not found. retry in 882.798579ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:58.383943987Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6\") failed: Property not found. retry in 759.85758ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:58.395821808Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873\") failed: Property not found. retry in 219.038706ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:58.405992321Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744\") failed: Property not found. retry in 712.821372ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:58.418433051Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728\") failed: Property not found. retry in 170.170348ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:58.429158483Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9\") failed: Property not found. retry in 1.088044382s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:58.506856379Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728\") failed: Property not found. retry in 312.676992ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:58.613045857Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873\") failed: Property not found. retry in 359.228543ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:58.639407244Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e\") failed: Property not found. retry in 1.15466898s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:58.677690862Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d\") failed: Property not found. retry in 1.477337458s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:58.741851919Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37\") failed: Property not found. retry in 715.262962ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:58.826899059Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473\") failed: Property not found. retry in 682.244ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:58.898494041Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728\") failed: Property not found. retry in 1.144646747s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:58.935175082Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873\") failed: Property not found. retry in 809.697437ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:59.010669591Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7\") failed: Property not found. retry in 784.306404ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:59.03067723Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257\") failed: Property not found. retry in 1.077028745s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:59.098524399Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744\") failed: Property not found. retry in 772.39999ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:59.142143598Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6\") failed: Property not found. retry in 1.436884671s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:59.166894427Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18\") failed: Property not found. retry in 1.175145417s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:59.185774446Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb\") failed: Property not found. retry in 546.503736ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:59.278481176Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41\") failed: Property not found. retry in 1.069923725s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:59.38025685Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6\") failed: Property not found. retry in 1.305471122s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:59.406657427Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37\") failed: Property not found. retry in 656.653776ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:59.554017362Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473\") failed: Property not found. retry in 1.258387701s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:59.594495748Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100\") failed: Property not found. retry in 1.403127027s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:59.605167499Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9\") failed: Property not found. retry in 1.028282801s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:59.617510657Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7\") failed: Property not found. retry in 987.112309ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:59.829473187Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb\") failed: Property not found. retry in 834.997202ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:59.842504608Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873\") failed: Property not found. retry in 1.406171506s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:59.854907108Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e\") failed: Property not found. retry in 698.356998ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:59.868412737Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744\") failed: Property not found. retry in 529.686296ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:56:59.929234247Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728\") failed: Property not found. retry in 1.091758997s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:00.093814784Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257\") failed: Property not found. retry in 1.117006387s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:00.189364462Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37\") failed: Property not found. retry in 1.397439517s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:00.224373575Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d\") failed: Property not found. retry in 894.891002ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:00.236729314Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18\") failed: Property not found. retry in 849.583845ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:00.374570868Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41\") failed: Property not found. retry in 1.187037163s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:00.459921103Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744\") failed: Property not found. retry in 596.52734ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:00.468398597Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e\") failed: Property not found. retry in 1.382906434s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:00.576002419Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6\") failed: Property not found. retry in 1.085587954s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:00.613110718Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9\") failed: Property not found. retry in 789.786884ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:00.656590104Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb\") failed: Property not found. retry in 642.742441ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:00.68703116Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6\") failed: Property not found. retry in 873.847722ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:00.721049056Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7\") failed: Property not found. retry in 1.436609456s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:00.8264154Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473\") failed: Property not found. retry in 753.162774ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:00.861552075Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100\") failed: Property not found. retry in 599.995479ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:01.017084789Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744\") failed: Property not found. retry in 534.590173ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:01.074573912Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d\") failed: Property not found. retry in 1.346970122s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:01.140821304Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728\") failed: Property not found. retry in 1.45975504s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:01.195262526Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18\") failed: Property not found. retry in 1.050654824s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:01.232456146Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873\") failed: Property not found. retry in 1.161935992s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:01.272727928Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257\") failed: Property not found. retry in 1.0456646s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:01.337589497Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb\") failed: Property not found. retry in 1.360801899s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:01.353041588Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9\") failed: Property not found. retry in 1.177302205s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:01.459283118Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6\") failed: Property not found. retry in 1.291872425s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:01.608495424Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744\") failed: Property not found. retry in 1.257490302s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:01.619293447Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100\") failed: Property not found. retry in 1.181565814s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:01.627404605Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473\") failed: Property not found. retry in 952.911951ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:01.636998087Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37\") failed: Property not found. retry in 1.32745199s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:01.646326095Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41\") failed: Property not found. retry in 664.181405ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:01.65842192Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6\") failed: Property not found. retry in 1.051603106s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:01.714936137Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e\") failed: Property not found. retry in 560.135893ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:01.971696365Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:02.060741053Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_external_dns_b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:02.06112227Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7\") failed: Property not found. retry in 756.83382ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:02.276165031Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18\") failed: Property not found. retry in 1.230950972s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:02.296265112Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41\") failed: Property not found. retry in 1.3524135s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:02.336188288Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac\") failed: Property not found. retry in 52.217109ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:02.384408424Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257\") failed: Property not found. retry in 1.461253796s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:02.395538513Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873\") failed: Property not found. retry in 607.427304ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:02.447493393Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac\") failed: Property not found. retry in 112.905715ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:02.456978823Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d\") failed: Property not found. retry in 1.057177066s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:02.500127534Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e\") failed: Property not found. retry in 1.493872832s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:02.544296252Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac\") failed: Property not found. retry in 205.789895ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:02.5802008Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473\") failed: Property not found. retry in 1.447352865s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:02.600762898Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9\") failed: Property not found. retry in 1.465671937s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:02.646529923Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728\") failed: Property not found. retry in 1.148377585s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:02.665152559Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb\") failed: Property not found. retry in 1.130969384s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:02.725417499Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6\") failed: Property not found. retry in 579.387719ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:02.780146076Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac\") failed: Property not found. retry in 529.309842ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:02.796810678Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100\") failed: Property not found. retry in 1.150051075s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:02.822819289Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744\") failed: Property not found. retry in 707.361368ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:02.896803552Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6\") failed: Property not found. retry in 968.343548ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:02.913575136Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37\") failed: Property not found. retry in 1.280458646s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:02.983737504Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7\") failed: Property not found. retry in 1.288992321s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:03.042647792Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_2c94012a-643d-4f61-8185-c8786e00e320:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:03.044566544Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_nexus_2c94012a-643d-4f61-8185-c8786e00e320 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:03.0449086Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_2c94012a-643d-4f61-8185-c8786e00e320","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873\") failed: Property not found. retry in 1.290851971s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:03.065161153Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2c94012a-643d-4f61-8185-c8786e00e320\") failed: Property not found. retry in 74.749438ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:03.197568125Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_2c94012a-643d-4f61-8185-c8786e00e320","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2c94012a-643d-4f61-8185-c8786e00e320\") failed: Property not found. retry in 113.191613ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:03.281376684Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_2c94012a-643d-4f61-8185-c8786e00e320","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac\") failed: Property not found. retry in 537.745829ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:03.335872817Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6\") failed: Property not found. retry in 1.024906976s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:03.370212599Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2c94012a-643d-4f61-8185-c8786e00e320\") failed: Property not found. retry in 228.222103ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:03.408225432Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_2c94012a-643d-4f61-8185-c8786e00e320","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18\") failed: Property not found. retry in 1.215361937s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:03.537307288Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d\") failed: Property not found. retry in 1.178149667s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:03.567131903Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744\") failed: Property not found. retry in 602.076772ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:03.613715652Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2c94012a-643d-4f61-8185-c8786e00e320\") failed: Property not found. retry in 449.396612ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:03.645375019Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_2c94012a-643d-4f61-8185-c8786e00e320","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41\") failed: Property not found. retry in 1.181484743s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:03.698398637Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728\") failed: Property not found. retry in 504.440473ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:03.823039687Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb\") failed: Property not found. retry in 659.251782ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:03.86628885Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257\") failed: Property not found. retry in 1.176473734s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:03.874519369Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac\") failed: Property not found. retry in 730.295413ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:03.882802899Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6\") failed: Property not found. retry in 951.351039ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:03.890843025Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100\") failed: Property not found. retry in 940.578068ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:03.982465667Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e\") failed: Property not found. retry in 783.348553ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:04.047525059Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473\") failed: Property not found. retry in 560.483455ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:04.056850777Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2c94012a-643d-4f61-8185-c8786e00e320\") failed: Property not found. retry in 617.667682ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:04.105662554Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_2c94012a-643d-4f61-8185-c8786e00e320","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9\") failed: Property not found. retry in 1.057265152s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:04.120964443Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744\") failed: Property not found. retry in 770.662492ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:04.225109046Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37\") failed: Property not found. retry in 779.257811ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:04.271813237Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728\") failed: Property not found. retry in 1.188836519s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:04.334511439Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7\") failed: Property not found. retry in 1.189925694s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:04.341217262Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873\") failed: Property not found. retry in 1.476205283s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:04.365012065Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6\") failed: Property not found. retry in 1.35900322s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:04.402964348Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb\") failed: Property not found. retry in 585.816331ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:04.534260781Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac\") failed: Property not found. retry in 1.165364824s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:04.621965517Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473\") failed: Property not found. retry in 1.093664176s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:04.630607233Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2c94012a-643d-4f61-8185-c8786e00e320\") failed: Property not found. retry in 984.882059ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:04.73137703Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_2c94012a-643d-4f61-8185-c8786e00e320","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d\") failed: Property not found. retry in 827.386796ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:04.753913331Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18\") failed: Property not found. retry in 1.216343275s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:04.762535907Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e\") failed: Property not found. retry in 1.405282241s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:04.840442707Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6\") failed: Property not found. retry in 1.269891922s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:04.850769622Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41\") failed: Property not found. retry in 786.233189ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:04.889312104Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100\") failed: Property not found. retry in 923.083571ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:04.931306376Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744\") failed: Property not found. retry in 1.335057307s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:05.004413263Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37\") failed: Property not found. retry in 1.10363099s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:05.059843652Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257\") failed: Property not found. retry in 836.727787ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:05.067440971Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb\") failed: Property not found. retry in 839.434422ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:05.128654047Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9\") failed: Property not found. retry in 1.106412201s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:05.18725314Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728\") failed: Property not found. retry in 1.068295174s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:05.532176831Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7\") failed: Property not found. retry in 737.332826ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:05.540042304Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d\") failed: Property not found. retry in 1.254892634s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:05.589594453Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41\") failed: Property not found. retry in 1.019759977s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:05.685744672Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2c94012a-643d-4f61-8185-c8786e00e320\") failed: Property not found. retry in 618.520373ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:05.725473194Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_2c94012a-643d-4f61-8185-c8786e00e320","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473\") failed: Property not found. retry in 983.288287ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:05.733451519Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6\") failed: Property not found. retry in 1.242112295s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:05.771146068Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac\") failed: Property not found. retry in 1.161398074s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:05.796296524Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873\") failed: Property not found. retry in 547.901698ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:05.849897231Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100\") failed: Property not found. retry in 521.532961ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:05.864290715Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257\") failed: Property not found. retry in 783.728236ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:05.913631341Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb\") failed: Property not found. retry in 716.660353ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:05.977093055Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18\") failed: Property not found. retry in 1.220163039s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:05.988039991Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6\") failed: Property not found. retry in 1.44413171s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:06.129299333Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37\") failed: Property not found. retry in 824.291655ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:06.172284091Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e\") failed: Property not found. retry in 1.322493761s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:06.256588639Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7\") failed: Property not found. retry in 1.177979982s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:06.285896405Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9\") failed: Property not found. retry in 1.252200607s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:06.301921086Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744\") failed: Property not found. retry in 1.42787529s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:06.346872568Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2c94012a-643d-4f61-8185-c8786e00e320\") failed: Property not found. retry in 953.60322ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:06.353594181Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_2c94012a-643d-4f61-8185-c8786e00e320","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100\") failed: Property not found. retry in 626.929597ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:06.394405442Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873\") failed: Property not found. retry in 939.980595ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:06.405551511Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728\") failed: Property not found. retry in 1.241348667s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:06.608325405Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb\") failed: Property not found. retry in 1.406839186s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:06.701362921Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257\") failed: Property not found. retry in 1.47995334s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:06.709658441Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41\") failed: Property not found. retry in 1.187056755s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:06.716981115Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473\") failed: Property not found. retry in 1.047094537s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:06.725882776Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d\") failed: Property not found. retry in 902.064963ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:06.853583318Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac\") failed: Property not found. retry in 1.115184739s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:06.967251403Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37\") failed: Property not found. retry in 629.455521ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:07.00657552Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6\") failed: Property not found. retry in 889.005939ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:07.022110383Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100\") failed: Property not found. retry in 1.259722463s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:07.030701178Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18\") failed: Property not found. retry in 948.356374ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:07.217682635Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2c94012a-643d-4f61-8185-c8786e00e320\") failed: Property not found. retry in 681.621078ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:07.316560989Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_2c94012a-643d-4f61-8185-c8786e00e320","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873\") failed: Property not found. retry in 1.04550559s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:07.35380051Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7\") failed: Property not found. retry in 1.49729033s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:07.47307574Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9\") failed: Property not found. retry in 1.023859928s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:07.563683304Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6\") failed: Property not found. retry in 926.488784ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:07.581618948Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e\") failed: Property not found. retry in 547.307297ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:07.589938299Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37\") failed: Property not found. retry in 811.242739ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:07.644837068Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d\") failed: Property not found. retry in 846.01893ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:07.766143283Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473\") failed: Property not found. retry in 1.473350252s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:07.782971968Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744\") failed: Property not found. retry in 1.220043788s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:07.791870679Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728\") failed: Property not found. retry in 1.127441138s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:07.859101397Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41\") failed: Property not found. retry in 938.924268ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:07.913428747Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6\") failed: Property not found. retry in 623.085361ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:07.921014126Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2c94012a-643d-4f61-8185-c8786e00e320\") failed: Property not found. retry in 875.149965ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:08.008841763Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_2c94012a-643d-4f61-8185-c8786e00e320","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac\") failed: Property not found. retry in 1.400629979s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:08.091474892Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb\") failed: Property not found. retry in 1.372714069s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:08.116851672Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e\") failed: Property not found. retry in 1.027858855s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:08.146736308Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18\") failed: Property not found. retry in 944.310269ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:08.185714898Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257\") failed: Property not found. retry in 826.866478ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:08.206878036Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100\") failed: Property not found. retry in 722.101487ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:08.303208608Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873\") failed: Property not found. retry in 1.090380512s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:08.411247717Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37\") failed: Property not found. retry in 1.415451695s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:08.465356144Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6\") failed: Property not found. retry in 840.078498ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:08.516803445Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6\") failed: Property not found. retry in 662.785965ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:08.552923777Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9\") failed: Property not found. retry in 698.338673ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:08.597182426Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d\") failed: Property not found. retry in 978.217833ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:08.620868427Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41\") failed: Property not found. retry in 1.003737651s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:08.86251647Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2c94012a-643d-4f61-8185-c8786e00e320\") failed: Property not found. retry in 741.765654ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:08.892386465Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_2c94012a-643d-4f61-8185-c8786e00e320","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7\") failed: Property not found. retry in 1.368539003s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:08.978367521Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728\") failed: Property not found. retry in 847.624711ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:08.996680862Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744\") failed: Property not found. retry in 966.280388ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:09.020606857Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100\") failed: Property not found. retry in 869.089993ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:09.032780203Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257\") failed: Property not found. retry in 715.160053ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:09.041575162Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18\") failed: Property not found. retry in 1.07362096s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:09.138163738Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e\") failed: Property not found. retry in 955.83774ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:09.184148866Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6\") failed: Property not found. retry in 1.292722633s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:09.224961577Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473\") failed: Property not found. retry in 614.707939ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:09.265562195Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9\") failed: Property not found. retry in 914.746495ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:09.303528338Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6\") failed: Property not found. retry in 1.259883858s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:09.365837223Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb\") failed: Property not found. retry in 1.30824049s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:09.498448059Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac\") failed: Property not found. retry in 1.39968391s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:09.505130062Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873\") failed: Property not found. retry in 1.168572595s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:09.511633402Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d\") failed: Property not found. retry in 1.216447565s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:09.611193298Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2c94012a-643d-4f61-8185-c8786e00e320\") failed: Property not found. retry in 1.142146449s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:09.645117223Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_2c94012a-643d-4f61-8185-c8786e00e320","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257\") failed: Property not found. retry in 1.47403258s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:09.768000484Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728\") failed: Property not found. retry in 858.488971ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:09.855098939Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41\") failed: Property not found. retry in 1.332654546s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:09.877992326Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473\") failed: Property not found. retry in 750.843294ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:09.891679888Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37\") failed: Property not found. retry in 1.152002826s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:09.901550505Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100\") failed: Property not found. retry in 953.490465ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:09.912266567Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_clickhouse_8471ffba-0f68-4241-be3b-68581f848fb7:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:09.915866798Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_clickhouse_8471ffba-0f68-4241-be3b-68581f848fb7 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:09.91897979Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_8471ffba-0f68-4241-be3b-68581f848fb7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744\") failed: Property not found. retry in 654.881263ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:09.998311304Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_8471ffba-0f68-4241-be3b-68581f848fb7\") failed: Property not found. retry in 31.450823ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:10.118968717Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_8471ffba-0f68-4241-be3b-68581f848fb7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e\") failed: Property not found. retry in 1.155313829s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:10.153476921Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_8471ffba-0f68-4241-be3b-68581f848fb7\") failed: Property not found. retry in 101.20665ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:10.16281998Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_8471ffba-0f68-4241-be3b-68581f848fb7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18\") failed: Property not found. retry in 901.9141ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:10.225554572Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9\") failed: Property not found. retry in 1.477893651s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:10.246328084Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_8471ffba-0f68-4241-be3b-68581f848fb7\") failed: Property not found. retry in 260.356637ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:10.278747523Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_8471ffba-0f68-4241-be3b-68581f848fb7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7\") failed: Property not found. retry in 1.097013523s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:10.359275447Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6\") failed: Property not found. retry in 592.405339ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:10.528994221Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_8471ffba-0f68-4241-be3b-68581f848fb7\") failed: Property not found. retry in 211.906706ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:10.550643887Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_8471ffba-0f68-4241-be3b-68581f848fb7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6\") failed: Property not found. retry in 1.16967083s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:10.638157429Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473\") failed: Property not found. retry in 1.354577506s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:10.655259019Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744\") failed: Property not found. retry in 529.9013ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:10.66651323Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873\") failed: Property not found. retry in 1.149083916s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:10.690656019Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728\") failed: Property not found. retry in 1.141537332s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:10.725035511Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_8471ffba-0f68-4241-be3b-68581f848fb7\") failed: Property not found. retry in 634.358234ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:10.77634568Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_8471ffba-0f68-4241-be3b-68581f848fb7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2c94012a-643d-4f61-8185-c8786e00e320\") failed: Property not found. retry in 1.463825591s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:10.799804787Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_2c94012a-643d-4f61-8185-c8786e00e320","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb\") failed: Property not found. retry in 635.63057ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:10.818783788Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d\") failed: Property not found. retry in 1.272549518s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:10.839815764Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100\") failed: Property not found. retry in 626.05677ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:10.877074266Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac\") failed: Property not found. retry in 839.719219ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:10.918764572Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37\") failed: Property not found. retry in 1.044106397s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:11.065167181Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6\") failed: Property not found. retry in 1.419637077s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:11.133931816Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18\") failed: Property not found. retry in 885.976835ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:11.144971912Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744\") failed: Property not found. retry in 907.047419ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:11.20679637Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41\") failed: Property not found. retry in 1.219894574s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:11.222624558Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257\") failed: Property not found. retry in 629.813136ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:11.252825898Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e\") failed: Property not found. retry in 1.125543346s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:11.336790591Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_8471ffba-0f68-4241-be3b-68581f848fb7\") failed: Property not found. retry in 1.042870592s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:11.425193888Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_8471ffba-0f68-4241-be3b-68581f848fb7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb\") failed: Property not found. retry in 918.700498ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:11.468709945Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7\") failed: Property not found. retry in 1.310463105s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:11.484482232Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100\") failed: Property not found. retry in 516.920144ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:11.519152119Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9\") failed: Property not found. retry in 1.202074444s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:11.740286874Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac\") failed: Property not found. retry in 763.174013ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:11.772475669Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6\") failed: Property not found. retry in 1.430851417s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:11.822543647Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873\") failed: Property not found. retry in 1.058396612s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:11.855554796Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728\") failed: Property not found. retry in 634.161246ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:11.8818029Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257\") failed: Property not found. retry in 612.869806ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:11.899039852Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473\") failed: Property not found. retry in 1.105123856s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:12.027355545Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18\") failed: Property not found. retry in 954.216173ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:12.056868185Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100\") failed: Property not found. retry in 895.755887ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:12.086958375Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37\") failed: Property not found. retry in 1.465401479s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:12.124764775Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744\") failed: Property not found. retry in 927.622417ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:12.137804836Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d\") failed: Property not found. retry in 1.381906494s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:12.15043636Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2c94012a-643d-4f61-8185-c8786e00e320\") failed: Property not found. retry in 649.175067ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:12.280290258Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_2c94012a-643d-4f61-8185-c8786e00e320","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb\") failed: Property not found. retry in 596.004826ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:12.41620166Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41\") failed: Property not found. retry in 627.568597ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:12.46404753Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e\") failed: Property not found. retry in 649.056883ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:12.483409528Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_8471ffba-0f68-4241-be3b-68581f848fb7\") failed: Property not found. retry in 1.023895157s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:12.506032581Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_8471ffba-0f68-4241-be3b-68581f848fb7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257\") failed: Property not found. retry in 784.899047ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:12.53191135Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728\") failed: Property not found. retry in 1.058953914s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:12.550541795Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac\") failed: Property not found. retry in 568.210073ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:12.572182721Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6\") failed: Property not found. retry in 1.491044474s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:12.586954392Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7\") failed: Property not found. retry in 945.58034ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:12.810534448Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873\") failed: Property not found. retry in 544.926475ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:12.934151631Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2c94012a-643d-4f61-8185-c8786e00e320\") failed: Property not found. retry in 1.014690537s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:12.954277852Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_2c94012a-643d-4f61-8185-c8786e00e320","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9\") failed: Property not found. retry in 1.363389602s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:12.971809519Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100\") failed: Property not found. retry in 546.561548ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:12.998685755Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18\") failed: Property not found. retry in 1.044134909s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:13.031423809Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb\") failed: Property not found. retry in 1.317376056s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:13.054069602Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744\") failed: Property not found. retry in 555.819445ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:13.092197448Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41\") failed: Property not found. retry in 1.044535894s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:13.112782087Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e\") failed: Property not found. retry in 1.300531978s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:13.153096829Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473\") failed: Property not found. retry in 915.550116ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:13.18148402Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac\") failed: Property not found. retry in 555.820047ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:13.224772083Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6\") failed: Property not found. retry in 536.358839ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:13.268473493Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257\") failed: Property not found. retry in 1.159890804s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:13.33087025Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873\") failed: Property not found. retry in 1.476792516s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:13.499679099Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:13.504094283Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"wait for service Some(\"oxz_clickhouse_8471ffba-0f68-4241-be3b-68581f848fb7\") failed: Property not found. retry in 1.396234507s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:13.548916622Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_8471ffba-0f68-4241-be3b-68581f848fb7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100\") failed: Property not found. retry in 954.307119ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:13.565113607Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d\") failed: Property not found. retry in 555.25042ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:13.580346865Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37\") failed: Property not found. retry in 827.768122ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:13.605075844Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728\") failed: Property not found. retry in 770.453517ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:13.619773473Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744\") failed: Property not found. retry in 1.009927955s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:13.65978482Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7\") failed: Property not found. retry in 526.469848ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:13.769481348Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac\") failed: Property not found. retry in 831.452962ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:13.792665254Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6\") failed: Property not found. retry in 555.54604ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:13.817695774Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2c94012a-643d-4f61-8185-c8786e00e320\") failed: Property not found. retry in 1.477917537s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:13.980709925Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_2c94012a-643d-4f61-8185-c8786e00e320","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18\") failed: Property not found. retry in 1.142752412s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:14.088694284Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6\") failed: Property not found. retry in 1.173369647s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:14.102261033Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473\") failed: Property not found. retry in 820.67599ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:14.115773122Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d\") failed: Property not found. retry in 515.30964ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:14.146654005Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41\") failed: Property not found. retry in 740.049808ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:14.168654288Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7\") failed: Property not found. retry in 1.373676544s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:14.310133144Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9\") failed: Property not found. retry in 1.181862141s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:14.350010959Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb\") failed: Property not found. retry in 678.656607ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:14.385279216Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6\") failed: Property not found. retry in 679.601013ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:14.396980814Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728\") failed: Property not found. retry in 785.638062ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:14.407818008Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37\") failed: Property not found. retry in 711.631657ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:14.446402691Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e\") failed: Property not found. retry in 1.272544717s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:14.468364323Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257\") failed: Property not found. retry in 1.037496812s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:14.504115309Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100\") failed: Property not found. retry in 1.139516421s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:14.53255611Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac\") failed: Property not found. retry in 696.779283ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:14.638735069Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d\") failed: Property not found. retry in 730.261572ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:14.674300211Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744\") failed: Property not found. retry in 875.001688ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:14.689093801Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41\") failed: Property not found. retry in 1.366777599s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:14.924968266Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473\") failed: Property not found. retry in 1.086158906s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:14.950232104Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_8471ffba-0f68-4241-be3b-68581f848fb7\") failed: Property not found. retry in 1.180023032s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:14.963656911Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_8471ffba-0f68-4241-be3b-68581f848fb7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873\") failed: Property not found. retry in 1.143439667s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:14.992669892Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb\") failed: Property not found. retry in 1.476078179s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:15.079880659Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6\") failed: Property not found. retry in 1.460489041s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:15.096292497Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37\") failed: Property not found. retry in 919.850576ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:15.175156393Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728\") failed: Property not found. retry in 580.207551ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:15.20925156Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18\") failed: Property not found. retry in 1.344909817s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:15.248786319Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6\") failed: Property not found. retry in 1.309868764s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:15.296697431Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac\") failed: Property not found. retry in 1.096876291s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:15.353375491Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d\") failed: Property not found. retry in 1.478290713s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:15.420776632Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2c94012a-643d-4f61-8185-c8786e00e320\") failed: Property not found. retry in 1.224395722s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:15.476008997Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_2c94012a-643d-4f61-8185-c8786e00e320","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9\") failed: Property not found. retry in 637.898802ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:15.545532445Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257\") failed: Property not found. retry in 880.484932ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:15.56176858Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744\") failed: Property not found. retry in 914.899163ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:15.580762671Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100\") failed: Property not found. retry in 1.184350711s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:15.689657035Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7\") failed: Property not found. retry in 906.894293ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:15.703190515Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e\") failed: Property not found. retry in 956.368569ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:15.755068253Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728\") failed: Property not found. retry in 1.135391106s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:15.802216692Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473\") failed: Property not found. retry in 1.230848759s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:16.058849138Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37\") failed: Property not found. retry in 1.333252589s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:16.113995301Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873\") failed: Property not found. retry in 701.773927ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:16.150891266Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_8471ffba-0f68-4241-be3b-68581f848fb7\") failed: Property not found. retry in 1.471430387s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:16.173105253Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_8471ffba-0f68-4241-be3b-68581f848fb7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9\") failed: Property not found. retry in 1.10262019s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:16.203216883Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41\") failed: Property not found. retry in 1.356646703s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:16.311605048Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257\") failed: Property not found. retry in 1.131921273s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:16.456909809Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac\") failed: Property not found. retry in 1.345945191s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:16.469932499Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744\") failed: Property not found. retry in 729.719508ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:16.515685364Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb\") failed: Property not found. retry in 743.303972ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:16.576512904Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6\") failed: Property not found. retry in 1.106961584s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:16.590835467Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18\") failed: Property not found. retry in 521.378468ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:16.610610952Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7\") failed: Property not found. retry in 1.480565814s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:16.62706731Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6\") failed: Property not found. retry in 950.72885ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:16.640905415Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2c94012a-643d-4f61-8185-c8786e00e320\") failed: Property not found. retry in 767.766907ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:16.716584266Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_2c94012a-643d-4f61-8185-c8786e00e320","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e\") failed: Property not found. retry in 529.367213ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:16.732932483Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873\") failed: Property not found. retry in 733.765181ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:16.867088135Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100\") failed: Property not found. retry in 1.040874403s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:16.890868498Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d\") failed: Property not found. retry in 1.47093322s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:16.913772596Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728\") failed: Property not found. retry in 647.027405ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:16.94947559Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18\") failed: Property not found. retry in 685.795028ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:17.144480483Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744\") failed: Property not found. retry in 714.78632ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:17.259621672Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e\") failed: Property not found. retry in 925.942036ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:17.275586283Z","hostname":"buskin","pid":1562,"zone":"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473\") failed: Property not found. retry in 532.042951ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:17.303699129Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9\") failed: Property not found. retry in 772.562436ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:17.318333077Z","hostname":"buskin","pid":1562,"zone":"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb\") failed: Property not found. retry in 1.030695229s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:17.332391445Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37\") failed: Property not found. retry in 1.228436544s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:17.461600913Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2c94012a-643d-4f61-8185-c8786e00e320\") failed: Property not found. retry in 563.437307ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:17.49741642Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_2c94012a-643d-4f61-8185-c8786e00e320","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257\") failed: Property not found. retry in 1.165135606s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:17.602376767Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6\") failed: Property not found. retry in 1.042213187s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:17.613868021Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728\") failed: Property not found. retry in 1.032474021s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:17.624795397Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_8471ffba-0f68-4241-be3b-68581f848fb7\") failed: Property not found. retry in 1.280199521s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:17.665194911Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_8471ffba-0f68-4241-be3b-68581f848fb7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41\") failed: Property not found. retry in 535.029713ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:17.690705863Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6\") failed: Property not found. retry in 736.651326ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:17.720945415Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18\") failed: Property not found. retry in 902.970702ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:17.847377746Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2c94012a-643d-4f61-8185-c8786e00e320\") failed: Property not found. retry in 871.27401ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:18.079414366Z","hostname":"buskin","pid":1562,"zone":"oxz_nexus_2c94012a-643d-4f61-8185-c8786e00e320","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7\") failed: Property not found. retry in 932.81124ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:18.134328426Z","hostname":"buskin","pid":1562,"zone":"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_8471ffba-0f68-4241-be3b-68581f848fb7\") failed: Property not found. retry in 853.469368ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T21:57:19.042122949Z","hostname":"buskin","pid":1562,"zone":"oxz_clickhouse_8471ffba-0f68-4241-be3b-68581f848fb7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:19.951000351Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_crucible_914d3516-8bfd-4006-87d8-91481730f9c7"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:19.951068502Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_external_dns_1db57e91-9837-462c-9f2c-1b4fca780b7e"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:19.951076722Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_crucible_92744278-0a6d-41bc-a502-b694867741c6"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:19.951082992Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_crucible_pantry_1bb815f5-1d70-43b2-a7b7-1bb132d87fd6"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:19.951089282Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_nexus_f9a2f81c-00cf-49a5-a96e-1c4dfeba1728"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:19.951095402Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_oximeter_ba8b1177-03a3-49b6-a6ee-b364652c50c9"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:19.951101422Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_crucible_3dafd3b1-1230-4db3-9961-2a4d53cc3a37"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:19.951122113Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_crucible_pantry_d9421cb5-f5bb-43e4-a2f0-337cff4f1100"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:19.951128393Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_external_dns_b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:19.951135223Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_nexus_2c94012a-643d-4f61-8185-c8786e00e320"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:19.951153243Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_crucible_66dd6deb-9c39-47ef-9f1c-98fea2467257"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:19.951159463Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_crucible_9c2bbea7-8b8c-4d83-9618-26c0848936eb"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:19.951166213Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_nexus_4e26237f-8cdc-4da1-b44a-9b00d20bd873"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:19.951172423Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_crucible_2bda41aa-8135-413a-853b-44e7fc746744"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:19.951178463Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_crucible_581275cf-e6a7-4ef5-ad1e-36b1379b2a18"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:19.951184934Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_crucible_pantry_8343e6aa-08fb-4b4a-bf94-a818d66d575d"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:19.951191254Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_clickhouse_8471ffba-0f68-4241-be3b-68581f848fb7"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:19.951203234Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_crucible_83653856-00c4-44fe-b4df-ad99e78af473"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:19.951209624Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245","zone":"oxz_crucible_02433f6a-f8a9-4f05-af1b-be59dfd00a41"} {"msg":"Writing ledger to /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:19.95273833Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:19.956019825Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:19.959272479Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"c3022fca-c201-49c6-86ba-ebdc97feba00","remote_addr":"[fd00:1122:3344:101::1]:55451","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":38222368,"response_code":"204"} {"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:19.988327023Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1286"} {"msg":"Writing ledger to /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:19.989401171Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:19.98992979Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:19.993870986Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:689"} {"msg":"Nexus address: [fd00:1122:3344:101::b]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:19.99941097Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:702"} {"msg":"rack_network_config: RackNetworkConfigV2 {\n bfd: [],\n bgp: [],\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV2 {\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n autoneg: false,\n bgp_peers: [],\n lldp: None,\n port: \"qsfp0\",\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n local_pref: None,\n nexthop: 10.151.1.100,\n vlan_id: None,\n },\n ],\n switch: Switch0,\n uplink_port_fec: None,\n uplink_port_speed: Speed40G,\n },\n ],\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:20.065947457Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:867"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/da7dff3e-1935-4d21-ab58-793c0302e3ea/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:20.068787585Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/da7dff3e-1935-4d21-ab58-793c0302e3ea/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:20.216740161Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/da7dff3e-1935-4d21-ab58-793c0302e3ea/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:20.282171429Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/da7dff3e-1935-4d21-ab58-793c0302e3ea/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:20.443625123Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/da7dff3e-1935-4d21-ab58-793c0302e3ea/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:20.543983322Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/da7dff3e-1935-4d21-ab58-793c0302e3ea/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:20.698462898Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/da7dff3e-1935-4d21-ab58-793c0302e3ea/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:20.878868624Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/da7dff3e-1935-4d21-ab58-793c0302e3ea/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:21.10537837Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/da7dff3e-1935-4d21-ab58-793c0302e3ea/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:21.403717982Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/da7dff3e-1935-4d21-ab58-793c0302e3ea/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:21.624147825Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/da7dff3e-1935-4d21-ab58-793c0302e3ea/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:22.279712616Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/da7dff3e-1935-4d21-ab58-793c0302e3ea/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:23.094363152Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/da7dff3e-1935-4d21-ab58-793c0302e3ea/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:23.771901736Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/da7dff3e-1935-4d21-ab58-793c0302e3ea/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:24.556567414Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/da7dff3e-1935-4d21-ab58-793c0302e3ea/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:25.940879108Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:26.048424179Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:61363"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:26.049318234Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"298145b0-dd78-4b8c-8df6-90688a668584","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:61363","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":161,"response_code":"200"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/da7dff3e-1935-4d21-ab58-793c0302e3ea/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:27.060401936Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/da7dff3e-1935-4d21-ab58-793c0302e3ea/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:28.549659337Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"76f06a97-d058-4f33-8f27-5c20bea0c352\", \"content-length\": \"173\", \"date\": \"Fri, 23 Aug 2024 21:57:30 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: rack with id \\\"da7dff3e-1935-4d21-ab58-793c0302e3ea\\\"\", request_id: \"76f06a97-d058-4f33-8f27-5c20bea0c352\" }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:30.648829026Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e2162715-de37-4bcd-bef2-9225dd65baa9\", \"content-length\": \"173\", \"date\": \"Fri, 23 Aug 2024 21:57:33 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"5050fe14-0845-40b2-872d-d70a5b2b56be\\\"\", request_id: \"e2162715-de37-4bcd-bef2-9225dd65baa9\" }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:34.309934495Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3f1757ef-0a45-4294-88df-e1bdd1c3add9\", \"content-length\": \"173\", \"date\": \"Fri, 23 Aug 2024 21:57:37 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"5050fe14-0845-40b2-872d-d70a5b2b56be\\\"\", request_id: \"3f1757ef-0a45-4294-88df-e1bdd1c3add9\" }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:37.744346147Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"42d65ef7-b71e-4c98-80b9-c89d7f055cd2\", \"content-length\": \"173\", \"date\": \"Fri, 23 Aug 2024 21:57:42 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"5050fe14-0845-40b2-872d-d70a5b2b56be\\\"\", request_id: \"42d65ef7-b71e-4c98-80b9-c89d7f055cd2\" }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:42.357919187Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8a0834e9-c1e2-4559-8232-2a8eeae3e80e\", \"content-length\": \"173\", \"date\": \"Fri, 23 Aug 2024 21:57:45 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"5050fe14-0845-40b2-872d-d70a5b2b56be\\\"\", request_id: \"8a0834e9-c1e2-4559-8232-2a8eeae3e80e\" }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:45.75231727Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0f68f113-2468-4510-a297-f321fb8c1053\", \"content-length\": \"173\", \"date\": \"Fri, 23 Aug 2024 21:57:51 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"5050fe14-0845-40b2-872d-d70a5b2b56be\\\"\", request_id: \"0f68f113-2468-4510-a297-f321fb8c1053\" }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:51.460323435Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:929"} {"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:54.998043526Z","hostname":"buskin","pid":1562,"component":"NexusNotifierTask","sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"sled-agent/src/nexus.rs:544","new_generation":"1","old_generation":"1"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:55.139585673Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:47633"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:55.142040274Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:55.232043118Z","hostname":"buskin","pid":1562,"uri":"/inventory","method":"GET","req_id":"10475df8-de1f-48a6-8d0e-9674da7d7605","remote_addr":"[fd00:1122:3344:101::a]:47633","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":92339,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:55.234071783Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:55.234158444Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:55.241257234Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:55.241837044Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"GET","req_id":"83dad523-2dbb-476b-a0ec-442f8a4b4296","remote_addr":"[fd00:1122:3344:101::a]:47633","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":9357,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:56.084646407Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39724"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:57:56.084700588Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"dab9ed0d-0968-4946-8fcf-e837b9c9f2d2","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39724","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":37,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:00.477857676Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:37532"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:00.482439393Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:671","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2a9b2cf7-2fb3-4597-871e-44332b4d82f2, kind: Service { id: 1db57e91-9837-462c-9f2c-1b4fca780b7e }, name: Name(\"external-dns-1db57e91-9837-462c-9f2c-1b4fca780b7e\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 24])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 5d1239c9-696e-4616-a2a6-82968e91a2e1, kind: Service { id: b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac }, name: Name(\"external-dns-b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 184])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 00000426-1f68-402a-a60e-4fa2f73c115f, kind: Service { id: 4e26237f-8cdc-4da1-b44a-9b00d20bd873 }, name: Name(\"nexus-4e26237f-8cdc-4da1-b44a-9b00d20bd873\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 190, 168])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 5cf061ea-987b-4986-8378-702c868c90c5, kind: Service { id: f9a2f81c-00cf-49a5-a96e-1c4dfeba1728 }, name: Name(\"nexus-f9a2f81c-00cf-49a5-a96e-1c4dfeba1728\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 196, 138])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b2284d48-b022-4af4-a2f6-3a691122eb62, kind: Service { id: 2c94012a-643d-4f61-8185-c8786e00e320 }, name: Name(\"nexus-2c94012a-643d-4f61-8185-c8786e00e320\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 253, 42])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:00.482646617Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:00.482680227Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:00.482696528Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:00.482707808Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:00.482720658Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:00.482729568Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:00.482763979Z","hostname":"buskin","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"f85bba36-4bbf-4c89-b667-b4b988b8b079","remote_addr":"[fd00:1122:3344:101::b]:37532","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":3550,"response_code":"204"} {"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:00.48401935Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:939"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:00.733422414Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:51725"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:00.734562713Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"21ae1afc-4005-454c-8cea-67697206034a","remote_addr":"[fd00:1122:3344:101::c]:51725","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1105,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:00.742528288Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:51045"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:00.742625439Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:00.836980868Z","hostname":"buskin","pid":1562,"uri":"/inventory","method":"GET","req_id":"641f5f76-b3bf-4dbc-9a02-7d460541d514","remote_addr":"[fd00:1122:3344:101::b]:51045","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":94343,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:00.83714669Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:46956"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:00.837155Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:63190"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:00.838188198Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:00.83828503Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:00.838417502Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:671","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2a9b2cf7-2fb3-4597-871e-44332b4d82f2, kind: Service { id: 1db57e91-9837-462c-9f2c-1b4fca780b7e }, name: Name(\"external-dns-1db57e91-9837-462c-9f2c-1b4fca780b7e\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 24])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 5d1239c9-696e-4616-a2a6-82968e91a2e1, kind: Service { id: b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac }, name: Name(\"external-dns-b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 184])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 00000426-1f68-402a-a60e-4fa2f73c115f, kind: Service { id: 4e26237f-8cdc-4da1-b44a-9b00d20bd873 }, name: Name(\"nexus-4e26237f-8cdc-4da1-b44a-9b00d20bd873\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 190, 168])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 5cf061ea-987b-4986-8378-702c868c90c5, kind: Service { id: f9a2f81c-00cf-49a5-a96e-1c4dfeba1728 }, name: Name(\"nexus-f9a2f81c-00cf-49a5-a96e-1c4dfeba1728\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 196, 138])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b2284d48-b022-4af4-a2f6-3a691122eb62, kind: Service { id: 2c94012a-643d-4f61-8185-c8786e00e320 }, name: Name(\"nexus-2c94012a-643d-4f61-8185-c8786e00e320\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 253, 42])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:00.838470063Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:671","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2a9b2cf7-2fb3-4597-871e-44332b4d82f2, kind: Service { id: 1db57e91-9837-462c-9f2c-1b4fca780b7e }, name: Name(\"external-dns-1db57e91-9837-462c-9f2c-1b4fca780b7e\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 24])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 5d1239c9-696e-4616-a2a6-82968e91a2e1, kind: Service { id: b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac }, name: Name(\"external-dns-b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 184])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 00000426-1f68-402a-a60e-4fa2f73c115f, kind: Service { id: 4e26237f-8cdc-4da1-b44a-9b00d20bd873 }, name: Name(\"nexus-4e26237f-8cdc-4da1-b44a-9b00d20bd873\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 190, 168])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 5cf061ea-987b-4986-8378-702c868c90c5, kind: Service { id: f9a2f81c-00cf-49a5-a96e-1c4dfeba1728 }, name: Name(\"nexus-f9a2f81c-00cf-49a5-a96e-1c4dfeba1728\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 196, 138])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b2284d48-b022-4af4-a2f6-3a691122eb62, kind: Service { id: 2c94012a-643d-4f61-8185-c8786e00e320 }, name: Name(\"nexus-2c94012a-643d-4f61-8185-c8786e00e320\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 253, 42])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:00.838489463Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:00.838506733Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:00.838535114Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:00.838556154Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:00.838736977Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Reading ledger from /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:00.838746927Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:00.838751297Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:00.838787098Z","hostname":"buskin","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"21f4be17-ddff-4b82-abe8-644b326f891b","remote_addr":"[fd00:1122:3344:101::c]:63190","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1455,"response_code":"204"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:00.838797288Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:00.838830999Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:00.838857139Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:00.838867539Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:00.83888685Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:00.83889972Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:00.83891842Z","hostname":"buskin","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"fc3109e4-d332-4bea-b954-174bfe423dd9","remote_addr":"[fd00:1122:3344:101::b]:46956","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1394,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:00.839023602Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"GET","req_id":"95300ecc-594b-456d-a904-d4b236735a54","remote_addr":"[fd00:1122:3344:101::b]:51045","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1772,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:00.840321164Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"d2c1aeb0-4e63-40ee-8f60-ecdb8907d331","remote_addr":"[fd00:1122:3344:101::c]:51725","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":3175,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:00.893982603Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:63332"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:00.894648994Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"0667731b-dd7b-4697-8dbb-4f4e87dc4594","remote_addr":"[fd00:1122:3344:101::c]:63332","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":637,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:01.159237935Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:47819"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:01.160997575Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:01.276400558Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:01.358891446Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"124fec58-2444-4214-8086-7f995e17f017","remote_addr":"[fd00:1122:3344:101::c]:47819","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":199570,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:01.359109279Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:63432"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:01.36031027Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"1c942b21-f2c7-4f4e-a38d-971d866990df","remote_addr":"[fd00:1122:3344:101::a]:63432","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1158,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:01.36799445Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"f426ad22-5e70-4de1-bd66-4ef87277a3a8","remote_addr":"[fd00:1122:3344:101::c]:47819","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":8654,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:01.369473105Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"eaf82a21-189c-40b6-a587-60f596a5e333","remote_addr":"[fd00:1122:3344:101::a]:63432","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":184,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:01.405513745Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:46720"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:01.405691258Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"f77e062d-6763-4466-aca6-8563b2703520","remote_addr":"[fd00:1122:3344:101::b]:46720","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":117,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:01.429778716Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:56988"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:01.430917035Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"cc9e4e5f-8a59-4960-8d52-d0e1bf068ad9","remote_addr":"[fd00:1122:3344:101::b]:56988","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1090,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:01.441477994Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"a1cdb777-01e7-473b-ba2a-274a1b9a62ec","remote_addr":"[fd00:1122:3344:101::b]:56988","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":80,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:01.506018377Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:42741"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:01.50620955Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"c243ebbd-d0b2-460a-bf34-f32527e9fc5a","remote_addr":"[fd00:1122:3344:101::a]:42741","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":146,"response_code":"200"} {"msg":"Attempting to update network config with generation: 2, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:01.506229751Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/156808b0-3956-455e-a8e1-2217e0afca54/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:01.506470115Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:01.508867546Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 2","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:01.510165267Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:01.510176497Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"PUT","req_id":"43fd1620-cc2d-4445-8787-e54ffc916e51","remote_addr":"[fd00:1122:3344:101::c]:47819","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":5954,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:01.579690465Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:44182"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:01.579771026Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:01.621412321Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:01.684261506Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"b764661d-827b-4442-9f2d-1d6afafb9744","remote_addr":"[fd00:1122:3344:101::b]:44182","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":104520,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:01.68448408Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:38526"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:01.684669153Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"aacbc3a5-10dd-4c82-8aa0-df66ec9a2507","remote_addr":"[fd00:1122:3344:101::b]:44182","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":146,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:01.685365955Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:01.725440093Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:01.785599092Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"0c44e61b-f089-4709-977b-1d81a93fa9bf","remote_addr":"[fd00:1122:3344:101::a]:38526","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":101010,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:01.78605905Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"fc0204bc-d4e0-47fe-ba15-92116ff79be4","remote_addr":"[fd00:1122:3344:101::a]:38526","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":173,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:02.066235374Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:58328"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:02.066519389Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:671","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2a9b2cf7-2fb3-4597-871e-44332b4d82f2, kind: Service { id: 1db57e91-9837-462c-9f2c-1b4fca780b7e }, name: Name(\"external-dns-1db57e91-9837-462c-9f2c-1b4fca780b7e\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 24])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 5d1239c9-696e-4616-a2a6-82968e91a2e1, kind: Service { id: b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac }, name: Name(\"external-dns-b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 184])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 00000426-1f68-402a-a60e-4fa2f73c115f, kind: Service { id: 4e26237f-8cdc-4da1-b44a-9b00d20bd873 }, name: Name(\"nexus-4e26237f-8cdc-4da1-b44a-9b00d20bd873\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 190, 168])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 5cf061ea-987b-4986-8378-702c868c90c5, kind: Service { id: f9a2f81c-00cf-49a5-a96e-1c4dfeba1728 }, name: Name(\"nexus-f9a2f81c-00cf-49a5-a96e-1c4dfeba1728\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 196, 138])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b2284d48-b022-4af4-a2f6-3a691122eb62, kind: Service { id: 2c94012a-643d-4f61-8185-c8786e00e320 }, name: Name(\"nexus-2c94012a-643d-4f61-8185-c8786e00e320\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 253, 42])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:02.06658602Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:02.066627131Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:02.066639711Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:02.066661282Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:02.066692072Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:02.066701802Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:02.066723783Z","hostname":"buskin","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"8053885b-d808-4e96-aee3-328aef377369","remote_addr":"[fd00:1122:3344:101::a]:58328","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":407,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:08.452553796Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:41348"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:08.452620537Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"cb85813b-0737-437f-b565-cb79e7ca1a9e","remote_addr":"[fd00:1122:3344:101::b]:41348","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":49,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:08.459082706Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"08f10cbd-04e6-4c11-b721-dbd8462bf7ef","remote_addr":"[fd00:1122:3344:101::b]:41348","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":96,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:09.218824303Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:46625","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","component":"producer-server","sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::d]:62985"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:09.282550022Z","hostname":"buskin","pid":1562,"uri":"/5050fe14-0845-40b2-872d-d70a5b2b56be","method":"GET","req_id":"505040f2-40fe-46fe-91a3-7d9fb5fa1f40","remote_addr":"[fd00:1122:3344:101::d]:62985","local_addr":"[fd00:1122:3344:101::1]:46625","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":63639,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:26.122952191Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:44519"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:26.123127034Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"20c5d00f-1c66-4594-9c45-afbe569492ec","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:44519","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":37,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:30.659550267Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:62129"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:30.659639999Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"289585a5-fcab-40f4-975a-534b4799ca47","remote_addr":"[fd00:1122:3344:101::c]:62129","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":79,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:30.77955225Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"b1d9b731-11e2-4ebb-af15-f1767fbf92cc","remote_addr":"[fd00:1122:3344:101::c]:62129","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":112,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:30.841816054Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:63637"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:30.841944306Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"54edfc61-89af-4abd-9d18-0fb18e6419a9","remote_addr":"[fd00:1122:3344:101::c]:63637","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":109,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:31.289595767Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:62158"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:31.289737249Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:31.333136514Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:31.395677583Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"60f10eec-f412-471f-9473-5c295127ecc4","remote_addr":"[fd00:1122:3344:101::c]:62158","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":106024,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:31.397173749Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"acd237aa-6855-4c75-bf6e-9572cb68ceaa","remote_addr":"[fd00:1122:3344:101::c]:62158","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1287,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:31.450699705Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:54205"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:31.450913689Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"e564e7bd-b2b5-4806-acde-760516d0d7fc","remote_addr":"[fd00:1122:3344:101::b]:54205","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":170,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:31.53131868Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:60128"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:31.531438213Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"6d1a9e50-f2f0-4c12-a04f-898ddb1e3527","remote_addr":"[fd00:1122:3344:101::a]:60128","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":64,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:31.541178478Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"db3db577-9393-42b0-9cb5-c5e5bef76283","remote_addr":"[fd00:1122:3344:101::a]:60128","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1148,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:31.613007484Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:41002"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:31.613083925Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"3fb6f394-4411-4cba-8e0e-b77e23452cba","remote_addr":"[fd00:1122:3344:101::b]:41002","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":47,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:31.622107468Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"31f7b90d-b3a9-4e0c-aaf0-4f7e0d97d670","remote_addr":"[fd00:1122:3344:101::b]:41002","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":85,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:31.628541217Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:45586"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:31.62870125Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"728f5502-a903-42d8-b1b0-52129a599f36","remote_addr":"[fd00:1122:3344:101::a]:45586","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":102,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:31.786139706Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:52537"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:31.786251588Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:31.827956284Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:31.883678858Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"8bd77a2f-4325-4aa0-b324-d01968f88a46","remote_addr":"[fd00:1122:3344:101::b]:52537","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":97480,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:31.88381734Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:34662"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:31.884004833Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:31.887313709Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"0dfb381b-bb07-4d7c-9eab-09e4d513bca2","remote_addr":"[fd00:1122:3344:101::b]:52537","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":3259,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:31.917603292Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:31.976906857Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"f0bcb50b-3fac-4a60-ba5d-1b218cd589e1","remote_addr":"[fd00:1122:3344:101::a]:34662","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":92988,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:31.977269313Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"4e9bffd7-fb06-41a0-9dfa-fabe28a8bbfc","remote_addr":"[fd00:1122:3344:101::a]:34662","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":162,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:39.227598115Z","hostname":"buskin","pid":1562,"uri":"/5050fe14-0845-40b2-872d-d70a5b2b56be","method":"GET","req_id":"072e7ee1-b3a8-430f-8d49-2805f6f88d84","remote_addr":"[fd00:1122:3344:101::d]:62985","local_addr":"[fd00:1122:3344:101::1]:46625","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":8260,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:56.159148159Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:34215"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:58:56.15922832Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"2022e32a-2be4-4240-adf6-c31137d9d665","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:34215","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":64,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:00.711157786Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:64481"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:00.711269528Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"82260bf6-2b45-4502-892e-6ad8dfbd3812","remote_addr":"[fd00:1122:3344:101::c]:64481","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":101,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:00.80823617Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"28af3017-a774-411c-8e6a-f6d4f5e882c2","remote_addr":"[fd00:1122:3344:101::c]:64481","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":151,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:00.889422365Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:40714"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:00.889606758Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"85b11506-22fb-4a53-9f86-5af68560dd2d","remote_addr":"[fd00:1122:3344:101::c]:40714","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":133,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:01.10348777Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:58029"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:01.104746312Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:01.153893814Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:01.229114918Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"9e9726e7-b043-4019-b107-c78a3de8160e","remote_addr":"[fd00:1122:3344:101::c]:58029","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":125551,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:01.230098704Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"debe5013-57ba-4ab2-ba09-c7cecd03009c","remote_addr":"[fd00:1122:3344:101::c]:58029","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":769,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:01.252189388Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:56431"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:01.253174205Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"ace96494-5527-4468-ad52-943f14cebca9","remote_addr":"[fd00:1122:3344:101::b]:56431","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":928,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:01.25993672Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"36412510-6696-48cf-9d74-bcbfcb7414c7","remote_addr":"[fd00:1122:3344:101::b]:56431","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":118,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:01.29773818Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:33328"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:01.297870832Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"81969f7d-641e-4489-bd85-71b4498b65da","remote_addr":"[fd00:1122:3344:101::b]:33328","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":113,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:01.419716805Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:58064"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:01.419841068Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:01.462720603Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:01.51798779Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"16ae5765-f887-4c08-ad04-6e46ada845fd","remote_addr":"[fd00:1122:3344:101::b]:58064","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":98200,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:01.518212763Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:57181"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:01.518321655Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:41720"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:01.518366846Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"f37b072b-3de8-46ef-8270-703f816865de","remote_addr":"[fd00:1122:3344:101::a]:57181","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":105,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:01.518515799Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"dc875cb3-88c5-4c64-bcbf-cc2e324bf9fd","remote_addr":"[fd00:1122:3344:101::b]:58064","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":270,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:01.518635421Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"56b74022-28da-47a5-b374-5e99f2d292e0","remote_addr":"[fd00:1122:3344:101::a]:41720","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":222,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:01.525865403Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"a4c87957-15e0-47f2-848e-3e6aa45cf419","remote_addr":"[fd00:1122:3344:101::a]:57181","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":662,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:01.553865627Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:49617"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:01.553940458Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:01.591261531Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:01.645912266Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"d1847014-ddbf-4030-bf57-3acdac229f9a","remote_addr":"[fd00:1122:3344:101::a]:49617","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":92008,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:01.646682039Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"48127ef7-5b65-40ce-8a4d-c4bc6eb81035","remote_addr":"[fd00:1122:3344:101::a]:49617","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":601,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:09.224511808Z","hostname":"buskin","pid":1562,"uri":"/5050fe14-0845-40b2-872d-d70a5b2b56be","method":"GET","req_id":"f0585996-fb03-47a6-a641-aa21cadd652a","remote_addr":"[fd00:1122:3344:101::d]:62985","local_addr":"[fd00:1122:3344:101::1]:46625","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":5676,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:26.19378841Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:57601"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:26.193873012Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"a9907fc8-b2c2-41d1-a579-ee38a1c2ef9b","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:57601","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":59,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:30.797667346Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:55384"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:30.798663203Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"7279c95d-155e-4e5a-9c2b-8ce7ec9d7891","remote_addr":"[fd00:1122:3344:101::c]:55384","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":984,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:30.805369326Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"d32481ca-b2c9-4c87-b867-5578a7eb22d5","remote_addr":"[fd00:1122:3344:101::c]:55384","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":115,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:30.818906836Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:46483"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:30.819053588Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"b92467d1-540e-48df-a8f7-12e5691502e7","remote_addr":"[fd00:1122:3344:101::c]:46483","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":101,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:31.133555674Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:48711"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:31.133685716Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:31.180962947Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:31.261160075Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"7d68cb4f-fb71-45ca-bc0b-41ab13f339c6","remote_addr":"[fd00:1122:3344:101::c]:48711","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":127514,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:31.261521451Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"4dc8f153-b3f1-49e5-94e3-48e9e23875be","remote_addr":"[fd00:1122:3344:101::c]:48711","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":118,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:31.370451216Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:47593"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:31.3706852Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"93bc566d-a9fd-480b-aa67-9e1cc44bbd8a","remote_addr":"[fd00:1122:3344:101::b]:47593","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":177,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:31.44391539Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:33544"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:31.444045272Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"cadf301f-f060-4779-b78f-b504f6d706ae","remote_addr":"[fd00:1122:3344:101::a]:33544","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":111,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:31.488490035Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:38023"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:31.488615767Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"af3848ab-4701-4e86-b2ba-096b40802e5b","remote_addr":"[fd00:1122:3344:101::b]:38023","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":53,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:31.496910827Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"bb91e5f7-eba6-4f85-8a8d-71272d2b6788","remote_addr":"[fd00:1122:3344:101::b]:38023","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":868,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:31.550621597Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:61805"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:31.550696369Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"6c1f91b8-7454-4323-9dc9-b6af6b20d636","remote_addr":"[fd00:1122:3344:101::a]:61805","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":40,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:31.556736301Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"4552a147-d495-4a1c-bb84-b3a817f359dd","remote_addr":"[fd00:1122:3344:101::a]:61805","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":81,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:31.736393233Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:51598"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:31.736496225Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:31.778792401Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:31.839367507Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"fa15e336-b4c1-428d-99fe-be7d5d269f00","remote_addr":"[fd00:1122:3344:101::b]:51598","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":102918,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:31.839679532Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"a2edcf11-a13e-4af4-89c4-8a6de43cef85","remote_addr":"[fd00:1122:3344:101::b]:51598","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":97,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:31.861092635Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:58950"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:31.861258048Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:31.901871276Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:31.962255618Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"fd604dcf-98eb-4da9-8c29-0f864ee27493","remote_addr":"[fd00:1122:3344:101::a]:58950","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":101061,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:31.962514083Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"a9876ee7-1094-4817-b279-26e076826304","remote_addr":"[fd00:1122:3344:101::a]:58950","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":112,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:39.22608502Z","hostname":"buskin","pid":1562,"uri":"/5050fe14-0845-40b2-872d-d70a5b2b56be","method":"GET","req_id":"37132e87-105e-4036-854e-7d02bc8ab99b","remote_addr":"[fd00:1122:3344:101::d]:62985","local_addr":"[fd00:1122:3344:101::1]:46625","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":6364,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:56.230269914Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63500"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T21:59:56.230968836Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"55944721-28dc-4a37-a17f-797a99d067cf","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63500","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":553,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:00.747087644Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:34896"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:00.747229527Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"368ac300-74ac-4b97-8bc4-5c7fba81cdfa","remote_addr":"[fd00:1122:3344:101::c]:34896","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":66,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:00.756615657Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"d75bc3b3-bf55-498c-b9f1-fae47b38872c","remote_addr":"[fd00:1122:3344:101::c]:34896","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":102,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:00.910462692Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:48257"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:00.911877546Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"ab1ae818-6e21-43e9-839b-b2d03678a9c1","remote_addr":"[fd00:1122:3344:101::c]:48257","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1386,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:01.090716175Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:56913"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:01.090844787Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:01.150175911Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:01.299270836Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"16e99220-e1c3-4821-b152-5f77afd3e1ad","remote_addr":"[fd00:1122:3344:101::c]:56913","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":208478,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:01.299459039Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:43168"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:01.299623362Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"89be85bd-c431-49ac-9447-30b5aa8a2505","remote_addr":"[fd00:1122:3344:101::b]:43168","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":115,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:01.299801655Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"72109ad0-0a83-4523-9387-86de31832a50","remote_addr":"[fd00:1122:3344:101::c]:56913","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":232,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:01.308521933Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"b1f2de5f-8d8f-439e-a2bf-91b2b9887f98","remote_addr":"[fd00:1122:3344:101::b]:43168","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":115,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:01.310846722Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:63671"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:01.310930604Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"85679d77-52de-4030-ac68-b85ef6892063","remote_addr":"[fd00:1122:3344:101::a]:63671","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":32,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:01.319204634Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"2d7168a8-893e-4a43-a5ae-5ebd8ea0c7fe","remote_addr":"[fd00:1122:3344:101::a]:63671","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":76,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:01.319419357Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:47417"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:01.320594947Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"b09bd784-8437-4497-82af-04e3a299d98d","remote_addr":"[fd00:1122:3344:101::b]:47417","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1121,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:01.425597206Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:41236"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:01.425753908Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"e4abdb91-7875-4d89-9879-e2f7bc94dedb","remote_addr":"[fd00:1122:3344:101::a]:41236","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":134,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:01.490444844Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:50843"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:01.490579206Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:01.578270431Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:01.718415284Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"f7546ce0-37ba-4721-98ff-d2e9dc33d465","remote_addr":"[fd00:1122:3344:101::b]:50843","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":227882,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:01.718639448Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:50326"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:01.718858522Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:01.718955903Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"85587396-d9d1-4754-a610-e152ad2276eb","remote_addr":"[fd00:1122:3344:101::b]:50843","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":257,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:01.801181736Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:01.929903716Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"1750f958-8097-4fe3-a2f1-7330ce279ad0","remote_addr":"[fd00:1122:3344:101::a]:50326","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":211150,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:01.930329933Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"a5df4eca-0677-4602-899d-a18a39c83560","remote_addr":"[fd00:1122:3344:101::a]:50326","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":168,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:09.23880534Z","hostname":"buskin","pid":1562,"uri":"/5050fe14-0845-40b2-872d-d70a5b2b56be","method":"GET","req_id":"54aaccd8-3290-40c4-9987-82e22e38b3ad","remote_addr":"[fd00:1122:3344:101::d]:62985","local_addr":"[fd00:1122:3344:101::1]:46625","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":7917,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:26.263628191Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63073"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:26.263721703Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"1386fa99-0632-4d2c-9f21-f47b8480fa3a","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63073","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":57,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:30.797089854Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:42824"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:30.797290248Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"622bd161-1d1c-4896-96a6-572956825dad","remote_addr":"[fd00:1122:3344:101::c]:42824","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":178,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:30.889813725Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:41689"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:30.889888306Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"ad2df47d-9b07-4a3e-90d3-39f6d4dece58","remote_addr":"[fd00:1122:3344:101::c]:41689","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":57,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:30.896703501Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"12436475-5014-4278-8b82-a6821f5b48f7","remote_addr":"[fd00:1122:3344:101::c]:41689","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":103,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:31.040870923Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:34909"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:31.041525464Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:31.084176066Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:31.154822043Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"727bf345-ddde-4257-b18f-74f7620998cd","remote_addr":"[fd00:1122:3344:101::c]:34909","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":113907,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:31.155174359Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"7884615f-32f9-4f96-952b-de56abff73c4","remote_addr":"[fd00:1122:3344:101::c]:34909","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":120,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:31.354293681Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:64482"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:31.354460863Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"86d88942-19b0-464d-8ecc-b89017766c55","remote_addr":"[fd00:1122:3344:101::b]:64482","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":113,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:31.36133753Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"983d6885-6ec2-46d6-8620-083dfafa2e4a","remote_addr":"[fd00:1122:3344:101::b]:64482","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":123,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:31.542347295Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:57958"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:31.542498788Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"faf75f59-1ef5-4a12-95ee-f9e76f1c89e8","remote_addr":"[fd00:1122:3344:101::b]:57958","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":123,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:31.571516839Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:55956"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:31.571671502Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"5d713121-a546-4c6a-a711-6e4e6ebe949e","remote_addr":"[fd00:1122:3344:101::a]:55956","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":94,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:31.668025183Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:50283"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:31.668115135Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"89be7e95-d6d0-48f3-bd34-e20c54aa8e0e","remote_addr":"[fd00:1122:3344:101::a]:50283","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":82,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:31.67548121Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"c3a25339-2c32-4c33-bbc9-ff1d375d2eb0","remote_addr":"[fd00:1122:3344:101::a]:50283","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":917,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:31.695688332Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:46407"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:31.695767844Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:31.735229392Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:31.790746792Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"3df64379-d2c7-4e12-9a33-f8d114e7923b","remote_addr":"[fd00:1122:3344:101::b]:46407","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":95014,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:31.790938765Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:50030"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:31.791123928Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:31.794713749Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"93bba67d-0d2d-4df5-9f4f-4e4b37bd2101","remote_addr":"[fd00:1122:3344:101::b]:46407","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":3583,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:31.830281482Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:31.887664263Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"d5d09ec8-b2a6-4843-9167-4ca7af45f689","remote_addr":"[fd00:1122:3344:101::a]:50030","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":96626,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:31.88805836Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"78024fa8-de22-4b9a-8ecc-e81975964173","remote_addr":"[fd00:1122:3344:101::a]:50030","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":180,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:39.279815148Z","hostname":"buskin","pid":1562,"uri":"/5050fe14-0845-40b2-872d-d70a5b2b56be","method":"GET","req_id":"11780133-9675-4471-8ab0-a8b1688702c9","remote_addr":"[fd00:1122:3344:101::d]:62985","local_addr":"[fd00:1122:3344:101::1]:46625","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":6846,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:56.297880596Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:59689"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:00:56.297959927Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"a1c3f2cd-9449-436f-8e16-5c1be5e8bb3e","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:59689","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":44,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:00.700008455Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:57241"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:00.700094276Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"ba974787-ce52-431e-a137-800ea90f8d14","remote_addr":"[fd00:1122:3344:101::c]:57241","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":86,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:00.707306418Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"613bd0ef-b19c-41da-b7b1-2ce3f719e9b4","remote_addr":"[fd00:1122:3344:101::c]:57241","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":126,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:00.820069578Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:45309"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:00.820903732Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"18d666ec-8f0b-4b25-8499-4b6aa7ec27ca","remote_addr":"[fd00:1122:3344:101::c]:45309","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":812,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:00.959844205Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:36439"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:00.959923147Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:00.999036289Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:01.05698646Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"19e5af2d-ac66-496a-acda-d167e1f4b483","remote_addr":"[fd00:1122:3344:101::c]:36439","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":97100,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:01.057259115Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"b6458829-e2f0-4b27-8ee8-3ca23f09289f","remote_addr":"[fd00:1122:3344:101::c]:36439","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":101,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:01.236412859Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:40177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:01.236585321Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"1347d64d-8341-4a5b-9f0c-7524918ff0f3","remote_addr":"[fd00:1122:3344:101::b]:40177","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":118,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:01.244320812Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"6030c7d3-11f1-442a-b063-b4e46f1cbd15","remote_addr":"[fd00:1122:3344:101::b]:40177","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":129,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:01.347193805Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:53153"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:01.348263173Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"6bae71c4-9028-4cda-ab86-3d3d124e71fb","remote_addr":"[fd00:1122:3344:101::b]:53153","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1053,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:01.384776551Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:46808"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:01.384915924Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"4f436dc5-e0a4-48f8-aa42-630f4afb8b28","remote_addr":"[fd00:1122:3344:101::a]:46808","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":87,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:01.401901141Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:41508"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:01.402027313Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"6bec0e7b-446f-4231-afe7-7cc4561a752b","remote_addr":"[fd00:1122:3344:101::a]:41508","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":57,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:01.408210528Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"e9e78032-370a-429e-bdac-950f62235d1c","remote_addr":"[fd00:1122:3344:101::a]:41508","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":99,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:01.439413387Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:54582"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:01.439495098Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:01.475447467Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:01.529099875Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"db94c4b7-c175-40e9-a593-6591c3b47615","remote_addr":"[fd00:1122:3344:101::b]:54582","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":89644,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:01.529238228Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:40222"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:01.529343939Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:01.564877391Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:01.621049812Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"e52c832d-20f0-49cf-b526-aa5296712a05","remote_addr":"[fd00:1122:3344:101::a]:40222","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":91728,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:01.621394488Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"57c808d9-397b-4bf3-8cab-7bfc606f2f19","remote_addr":"[fd00:1122:3344:101::b]:54582","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":141,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:01.621526351Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"b7ef718b-47ed-4bbf-bea8-99da2485d1c4","remote_addr":"[fd00:1122:3344:101::a]:40222","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":203,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:09.279493957Z","hostname":"buskin","pid":1562,"uri":"/5050fe14-0845-40b2-872d-d70a5b2b56be","method":"GET","req_id":"2d8299f8-dba6-48c7-98a6-936d22663f5b","remote_addr":"[fd00:1122:3344:101::d]:62985","local_addr":"[fd00:1122:3344:101::1]:46625","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":5898,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:26.331088752Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:64423"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:26.331791924Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"7441a8e9-e90d-4fb0-b7b3-c72dd928c3f7","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:64423","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":675,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:30.776854141Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:44062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:30.776931042Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"48a2f99a-f27b-437b-a1a2-e610c940fa6b","remote_addr":"[fd00:1122:3344:101::c]:44062","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":63,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:30.784258616Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"9fb6be63-c910-4b58-8f08-a98915deb538","remote_addr":"[fd00:1122:3344:101::c]:44062","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":751,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:30.807580251Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:33540"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:30.808493716Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"deb1d0bc-d61b-471c-a343-a2f34434227f","remote_addr":"[fd00:1122:3344:101::c]:33540","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":889,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:31.24268822Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:47972"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:31.242849222Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:31.283496521Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:31.342811445Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"7b071990-65e0-4a10-85f0-67cb8aeb96ea","remote_addr":"[fd00:1122:3344:101::c]:47972","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":100067,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:31.342982008Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:63657"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:31.34306238Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:50034"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:31.343246423Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"00dbe5c9-a2a3-4f5d-abb5-f215f53e69be","remote_addr":"[fd00:1122:3344:101::b]:63657","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":203,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:31.344400612Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"017ce25b-221f-4270-aed0-21c35da2d941","remote_addr":"[fd00:1122:3344:101::b]:50034","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1247,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:31.344527144Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"4d8c7060-f3ac-4999-929e-6618e62c0696","remote_addr":"[fd00:1122:3344:101::c]:47972","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1373,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:31.352097822Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"92fd3f3b-97f1-4b27-9d2d-01bcddb5978a","remote_addr":"[fd00:1122:3344:101::b]:50034","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":141,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:31.437788424Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:48984"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:31.437936646Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"0ebdf725-b7be-4064-a490-72424f40deda","remote_addr":"[fd00:1122:3344:101::a]:48984","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":98,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:31.554076093Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:47020"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:31.554175705Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"dcf43c44-0bbd-43da-89f1-b23f17566f0f","remote_addr":"[fd00:1122:3344:101::a]:47020","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":52,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:31.560229237Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"377898bd-d706-4b91-a38c-1fa3e83dbc67","remote_addr":"[fd00:1122:3344:101::a]:47020","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":89,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:31.581229683Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:36087"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:31.581313384Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:31.622754506Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:31.683426033Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"a7fdd03e-d288-48e1-8111-58df2d3c3f71","remote_addr":"[fd00:1122:3344:101::b]:36087","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":102156,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:31.684592463Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"08422356-81a5-4bbe-93a8-85861236105e","remote_addr":"[fd00:1122:3344:101::b]:36087","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":976,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:31.828926368Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:38090"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:31.829096431Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:31.869113058Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:31.931272681Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"cdbb9b2b-0cf2-4399-8ccb-f75bcf68f1d7","remote_addr":"[fd00:1122:3344:101::a]:38090","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":102249,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:31.932184056Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"1cbd7d67-e0a2-41a6-9574-9ff25abfa85e","remote_addr":"[fd00:1122:3344:101::a]:38090","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":731,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:39.279346559Z","hostname":"buskin","pid":1562,"uri":"/5050fe14-0845-40b2-872d-d70a5b2b56be","method":"GET","req_id":"06be3171-1d18-4e88-877c-5e970f3266c3","remote_addr":"[fd00:1122:3344:101::d]:62985","local_addr":"[fd00:1122:3344:101::1]:46625","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":6182,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:56.36432014Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:48486"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:56.364384681Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"0a691247-56e9-4954-8253-70f46e9a9dfb","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:48486","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":45,"response_code":"200"} {"msg":"running automatic periodic zone bundle cleanup","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:59.415889128Z","hostname":"buskin","pid":1562,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:197"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:59.416959676Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:01:59.459043188Z","hostname":"buskin","pid":1562,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:186","time_to_next_cleanup":"599.99999408s","next_cleanup":"Instant { tv_sec: 2670, tv_nsec: 206395128 }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:00.708523768Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:40629"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:00.70862633Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"559429c1-03af-42b1-a7e2-6cdfea14817a","remote_addr":"[fd00:1122:3344:101::c]:40629","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":101,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:00.7097736Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:48356"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:00.710591683Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"0bc4d1cc-2cd6-4497-8e0e-75bfb07ab518","remote_addr":"[fd00:1122:3344:101::c]:48356","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":795,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:00.7151157Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"1ea2f9c7-8030-478a-b076-299163b8f379","remote_addr":"[fd00:1122:3344:101::c]:40629","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":100,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:00.87399043Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:36220"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:00.874074802Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:00.913423798Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:00.968408469Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"c2bb71c7-e3f7-428a-b217-eebc05ffab20","remote_addr":"[fd00:1122:3344:101::c]:36220","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":94371,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:00.969276934Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"df777377-ece6-47c4-a36d-d34889403b77","remote_addr":"[fd00:1122:3344:101::c]:36220","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":733,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:01.235750767Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:36621"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:01.235900539Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"68706430-f158-4120-a08f-304569d648da","remote_addr":"[fd00:1122:3344:101::b]:36621","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":102,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:01.24303442Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"d04b932b-d8d7-4946-baf4-8571666e60b3","remote_addr":"[fd00:1122:3344:101::b]:36621","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":874,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:01.285423188Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:33536"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:01.285604231Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"0aaff43e-a985-4dcd-9bf1-bce81447e5b9","remote_addr":"[fd00:1122:3344:101::b]:33536","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":137,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:01.377323734Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:33321"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:01.377482717Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"542f00eb-d5e0-478f-8bbc-a03b31219b3d","remote_addr":"[fd00:1122:3344:101::a]:33321","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":131,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:01.43497434Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:61854"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:01.435054872Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"f369963f-2a33-473d-b902-40f4b4c8952d","remote_addr":"[fd00:1122:3344:101::a]:61854","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":57,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:01.44084402Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"29bef824-994c-4dd3-9fb0-0bde4b952025","remote_addr":"[fd00:1122:3344:101::a]:61854","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":85,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:01.445199094Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:58106"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:01.445896265Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:01.485950353Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:01.543742322Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"ed0a945e-c911-4dc1-b842-bffc53c4ac8e","remote_addr":"[fd00:1122:3344:101::b]:58106","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":98484,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:01.543925265Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:57449"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:01.544061898Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:01.548409201Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"5aab92fc-b8d1-42f6-a235-b3aa457460c1","remote_addr":"[fd00:1122:3344:101::b]:58106","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":4320,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:01.58316438Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:01.643692425Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"e056727f-dbfb-4add-8762-0e226c7fae56","remote_addr":"[fd00:1122:3344:101::a]:57449","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":99689,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:01.644692942Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"580c2f7b-9f6a-449a-a590-ccb1634920da","remote_addr":"[fd00:1122:3344:101::a]:57449","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":804,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:09.280403371Z","hostname":"buskin","pid":1562,"uri":"/5050fe14-0845-40b2-872d-d70a5b2b56be","method":"GET","req_id":"915bc01d-b23e-4117-8ab8-6470411fd0b2","remote_addr":"[fd00:1122:3344:101::d]:62985","local_addr":"[fd00:1122:3344:101::1]:46625","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":6585,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:13.509108074Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:26.399835485Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:54055"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:26.399916327Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"1c7be9e7-92e3-4132-8f1d-04eb12d016f8","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:54055","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":45,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:30.758732472Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:56888"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:30.758885125Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"bb4d30a0-204f-411e-8c2e-4cdfb305ffe8","remote_addr":"[fd00:1122:3344:101::c]:56888","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":149,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:30.79816269Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:48134"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:30.798267892Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"37335ceb-c633-496f-a716-5fdab7e61070","remote_addr":"[fd00:1122:3344:101::c]:48134","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":74,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:30.805560905Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"40e0d441-03be-4db5-a5b9-b6bd5dd011f1","remote_addr":"[fd00:1122:3344:101::c]:48134","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":106,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:30.91984053Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:61256"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:30.920087305Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:671","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2a9b2cf7-2fb3-4597-871e-44332b4d82f2, kind: Service { id: 1db57e91-9837-462c-9f2c-1b4fca780b7e }, name: Name(\"external-dns-1db57e91-9837-462c-9f2c-1b4fca780b7e\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 24])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 5d1239c9-696e-4616-a2a6-82968e91a2e1, kind: Service { id: b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac }, name: Name(\"external-dns-b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 184])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 00000426-1f68-402a-a60e-4fa2f73c115f, kind: Service { id: 4e26237f-8cdc-4da1-b44a-9b00d20bd873 }, name: Name(\"nexus-4e26237f-8cdc-4da1-b44a-9b00d20bd873\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 190, 168])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 5cf061ea-987b-4986-8378-702c868c90c5, kind: Service { id: f9a2f81c-00cf-49a5-a96e-1c4dfeba1728 }, name: Name(\"nexus-f9a2f81c-00cf-49a5-a96e-1c4dfeba1728\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 196, 138])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b2284d48-b022-4af4-a2f6-3a691122eb62, kind: Service { id: 2c94012a-643d-4f61-8185-c8786e00e320 }, name: Name(\"nexus-2c94012a-643d-4f61-8185-c8786e00e320\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 253, 42])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:30.920140546Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:30.920173496Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:30.920194857Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:30.920217657Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:30.920237708Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:30.920244977Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:30.920279128Z","hostname":"buskin","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"2ca00716-b9e1-4428-b708-374e2ca29b84","remote_addr":"[fd00:1122:3344:101::c]:61256","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":349,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:31.196266812Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:48620"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:31.197808098Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:31.249747468Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:31.322775674Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"1b328dae-a8cd-4cd7-a373-b1067bb031d5","remote_addr":"[fd00:1122:3344:101::c]:48620","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":126450,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:31.323323454Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"32f272da-c7f6-4ab6-97f1-f4f3e650b4dd","remote_addr":"[fd00:1122:3344:101::c]:48620","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":226,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:31.37802042Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:40138"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:31.378110351Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"c8eff402-ec69-4cd8-9688-f3553bdb6991","remote_addr":"[fd00:1122:3344:101::b]:40138","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":83,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:31.385012038Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"0a99da12-0735-4cea-8630-48aa92686930","remote_addr":"[fd00:1122:3344:101::b]:40138","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":112,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:31.390367449Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:55769"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:31.390519121Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"5f4db4fb-fa8f-4db5-9eff-0d5d7cd10184","remote_addr":"[fd00:1122:3344:101::b]:55769","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":116,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:31.410681273Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:53771"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:31.410861976Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:671","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2a9b2cf7-2fb3-4597-871e-44332b4d82f2, kind: Service { id: 1db57e91-9837-462c-9f2c-1b4fca780b7e }, name: Name(\"external-dns-1db57e91-9837-462c-9f2c-1b4fca780b7e\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 24])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 5d1239c9-696e-4616-a2a6-82968e91a2e1, kind: Service { id: b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac }, name: Name(\"external-dns-b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 184])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 00000426-1f68-402a-a60e-4fa2f73c115f, kind: Service { id: 4e26237f-8cdc-4da1-b44a-9b00d20bd873 }, name: Name(\"nexus-4e26237f-8cdc-4da1-b44a-9b00d20bd873\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 190, 168])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 5cf061ea-987b-4986-8378-702c868c90c5, kind: Service { id: f9a2f81c-00cf-49a5-a96e-1c4dfeba1728 }, name: Name(\"nexus-f9a2f81c-00cf-49a5-a96e-1c4dfeba1728\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 196, 138])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b2284d48-b022-4af4-a2f6-3a691122eb62, kind: Service { id: 2c94012a-643d-4f61-8185-c8786e00e320 }, name: Name(\"nexus-2c94012a-643d-4f61-8185-c8786e00e320\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 253, 42])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:31.410897677Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:31.410934607Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:31.410953268Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:31.41097147Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:31.41098272Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:31.410989038Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:31.411003018Z","hostname":"buskin","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"38f3a84a-c47e-457d-8d26-3fca731f86c3","remote_addr":"[fd00:1122:3344:101::b]:53771","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":275,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:31.450175032Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:46372"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:31.450295424Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"89d310c5-b916-4a76-87ef-0f6989392be1","remote_addr":"[fd00:1122:3344:101::a]:46372","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":97,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:31.460260163Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:42688"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:31.460312923Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"346e21c5-1578-48ac-a8db-c1f044c7e95d","remote_addr":"[fd00:1122:3344:101::a]:42688","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":37,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:31.467107549Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"0236898b-078b-4126-9acf-5c43732f5c60","remote_addr":"[fd00:1122:3344:101::a]:42688","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":74,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:31.67148755Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:61852"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:31.671667323Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:671","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2a9b2cf7-2fb3-4597-871e-44332b4d82f2, kind: Service { id: 1db57e91-9837-462c-9f2c-1b4fca780b7e }, name: Name(\"external-dns-1db57e91-9837-462c-9f2c-1b4fca780b7e\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 24])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 5d1239c9-696e-4616-a2a6-82968e91a2e1, kind: Service { id: b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac }, name: Name(\"external-dns-b28fec8f-a1e5-40b2-a907-cc5bb6eb8cac\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 184])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 00000426-1f68-402a-a60e-4fa2f73c115f, kind: Service { id: 4e26237f-8cdc-4da1-b44a-9b00d20bd873 }, name: Name(\"nexus-4e26237f-8cdc-4da1-b44a-9b00d20bd873\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 190, 168])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 5cf061ea-987b-4986-8378-702c868c90c5, kind: Service { id: f9a2f81c-00cf-49a5-a96e-1c4dfeba1728 }, name: Name(\"nexus-f9a2f81c-00cf-49a5-a96e-1c4dfeba1728\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 196, 138])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b2284d48-b022-4af4-a2f6-3a691122eb62, kind: Service { id: 2c94012a-643d-4f61-8185-c8786e00e320 }, name: Name(\"nexus-2c94012a-643d-4f61-8185-c8786e00e320\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 253, 42])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:31.671709834Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:31.671741214Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:31.671753524Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:31.671768935Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:31.671779525Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:31.671789855Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:690","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:31.671812695Z","hostname":"buskin","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"1724bc1e-7814-4e02-9926-d07818149946","remote_addr":"[fd00:1122:3344:101::a]:61852","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":307,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:31.675778623Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:49458"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:31.675862864Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:31.716783847Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:31.776282694Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"9a14ba3a-5b88-4f53-a1cd-0167a2865013","remote_addr":"[fd00:1122:3344:101::b]:49458","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":100458,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:31.77719458Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"feb6767c-91f5-4078-8e94-888e5ac491d2","remote_addr":"[fd00:1122:3344:101::b]:49458","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":735,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:31.916904356Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:35609"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:31.917096479Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:31.956457686Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:32.012920442Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"dffd2df6-8976-4bbe-a1ae-2e52579173f1","remote_addr":"[fd00:1122:3344:101::a]:35609","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":95911,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:32.013207277Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"5e7d744a-6f6a-4ff1-b73d-2f9485c80adf","remote_addr":"[fd00:1122:3344:101::a]:35609","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":105,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:39.280818922Z","hostname":"buskin","pid":1562,"uri":"/5050fe14-0845-40b2-872d-d70a5b2b56be","method":"GET","req_id":"df0d9f71-51ef-44a2-af80-b3ed15f051b1","remote_addr":"[fd00:1122:3344:101::d]:62985","local_addr":"[fd00:1122:3344:101::1]:46625","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":6434,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:56.434306823Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:65403"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:02:56.434376605Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"8c2aa861-db27-4fcd-baf6-00c72c3a2c89","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:65403","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":42,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:00.726062013Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:48531"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:00.726218936Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"01796382-3600-4d7f-999a-4ffc520beffe","remote_addr":"[fd00:1122:3344:101::c]:48531","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":143,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:00.877334475Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:50317"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:00.877407787Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"759fff6e-f77d-4503-a603-3741a17ca760","remote_addr":"[fd00:1122:3344:101::c]:50317","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":40,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:00.883828875Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"425438ce-65f8-48e1-9985-5806bc284060","remote_addr":"[fd00:1122:3344:101::c]:50317","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":76,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:00.955028931Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:43888"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:00.955087492Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:00.993066295Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:01.046457289Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"7713d984-b139-4d83-820d-a1cba1700017","remote_addr":"[fd00:1122:3344:101::c]:43888","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":91386,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:01.046845666Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"477dc818-d12f-4c5b-ade4-a12a79216493","remote_addr":"[fd00:1122:3344:101::c]:43888","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":99,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:01.294145644Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:37514"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:01.294285326Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"cb54e940-ea4f-438e-943f-d348b2e7605c","remote_addr":"[fd00:1122:3344:101::b]:37514","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":129,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:01.299984103Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:44120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:01.300839127Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"e32f908d-529f-4c7f-ba9c-b25d9d288732","remote_addr":"[fd00:1122:3344:101::a]:44120","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":837,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:01.30101451Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"9e04c136-de3f-4831-a936-61a88a09fbf6","remote_addr":"[fd00:1122:3344:101::b]:37514","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":120,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:01.306863639Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"b5dafc8e-2256-4405-896e-e91c63cef18f","remote_addr":"[fd00:1122:3344:101::a]:44120","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":65,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:01.315626128Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:51409"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:01.31577646Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"2cf2cf66-df8a-4676-a37a-e4b238adbe26","remote_addr":"[fd00:1122:3344:101::b]:51409","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":126,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:01.321538458Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:57617"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:01.32165207Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"029d2fd8-97c7-4915-a61e-b2682c1783c6","remote_addr":"[fd00:1122:3344:101::a]:57617","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":74,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:01.454253505Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:49535"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:01.454352537Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:01.493956321Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:01.555788275Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"155eb95c-49b3-4c29-a32e-9827cbecf527","remote_addr":"[fd00:1122:3344:101::a]:49535","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":101476,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:01.555984958Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:34975"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:01.55725452Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:01.557359951Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"3bb07a53-60d0-4515-8ce0-5d955a607708","remote_addr":"[fd00:1122:3344:101::a]:49535","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1219,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:01.593989392Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:01.649537682Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"602b3970-34ae-48e6-8ece-d53ed9fab389","remote_addr":"[fd00:1122:3344:101::b]:34975","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":93397,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:01.650418127Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"2771685a-dc25-4dd8-803b-a79533cda86e","remote_addr":"[fd00:1122:3344:101::b]:34975","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":659,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:09.39441578Z","hostname":"buskin","pid":1562,"uri":"/5050fe14-0845-40b2-872d-d70a5b2b56be","method":"GET","req_id":"8d6350df-403d-4b3d-bc7c-7739873e52f0","remote_addr":"[fd00:1122:3344:101::d]:62985","local_addr":"[fd00:1122:3344:101::1]:46625","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":7130,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:26.47286543Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63333"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:26.472942382Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"6f7842e8-1b68-484e-9b1d-01a591ca4157","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63333","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":56,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:30.794207052Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:46104"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:30.794309564Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"76210876-20c7-4f37-80db-b7927d850990","remote_addr":"[fd00:1122:3344:101::c]:46104","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":94,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:30.800745502Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"bf1569c8-4503-47c0-b8cc-ba613495db4f","remote_addr":"[fd00:1122:3344:101::c]:46104","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":120,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:30.936469951Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:37983"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:30.936622674Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"1418f9a4-3372-465f-956e-cea82ed35952","remote_addr":"[fd00:1122:3344:101::c]:37983","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":124,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:31.28307869Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:53387"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:31.283199423Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:31.32733579Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:31.390972428Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"ab136e5a-4857-4e43-b89b-213ba222b8e6","remote_addr":"[fd00:1122:3344:101::c]:53387","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":107827,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:31.391169351Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:52687"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:31.391405705Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"f57df826-dedb-4309-a510-3a25beb70538","remote_addr":"[fd00:1122:3344:101::b]:52687","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":203,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:31.391451366Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"6f300c1b-2eba-4c51-b8d1-b37a9554c3db","remote_addr":"[fd00:1122:3344:101::c]:53387","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":243,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:31.404215022Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:36954"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:31.405188688Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"b4f09ff9-eb67-47b6-a266-f0429259997a","remote_addr":"[fd00:1122:3344:101::a]:36954","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":943,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:31.411894242Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"18bf7f5a-0af6-4d28-a0a3-350a49b80457","remote_addr":"[fd00:1122:3344:101::a]:36954","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":129,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:31.419352118Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:40441"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:31.419496021Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"21e1db86-1634-447f-ab4a-bb8617a50249","remote_addr":"[fd00:1122:3344:101::a]:40441","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":125,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:31.459409647Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:46704"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:31.460442204Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"8c252da0-2ab1-4856-8094-21f806e31804","remote_addr":"[fd00:1122:3344:101::b]:46704","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1007,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:31.46668441Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"dbbf2619-3537-4294-8072-1f37e3655a7c","remote_addr":"[fd00:1122:3344:101::b]:46704","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":82,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:31.599015201Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:38421"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:31.599104052Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:31.638167744Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:31.700073712Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"c0caa9a7-28e4-4b80-b136-5f077f367393","remote_addr":"[fd00:1122:3344:101::b]:38421","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":101019,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:31.700342777Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"d31e8671-2b9b-4f17-b427-8a60abd4e018","remote_addr":"[fd00:1122:3344:101::b]:38421","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":95,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:31.719843037Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:47883"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:31.7200056Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:31.755679214Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:31.813971631Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"e888c2c7-0ce6-4101-9082-6a2d594b23c3","remote_addr":"[fd00:1122:3344:101::a]:47883","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":94030,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:31.814431369Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"e6a97142-6c45-4faf-9b51-1e313198c268","remote_addr":"[fd00:1122:3344:101::a]:47883","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":231,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:39.393426338Z","hostname":"buskin","pid":1562,"uri":"/5050fe14-0845-40b2-872d-d70a5b2b56be","method":"GET","req_id":"e746fd8d-9c1a-44c8-8295-c664ffc3ad22","remote_addr":"[fd00:1122:3344:101::d]:62985","local_addr":"[fd00:1122:3344:101::1]:46625","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":5793,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:56.505600329Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:44963"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:03:56.50565343Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"208ad112-e21c-4e15-be3b-c8b7bb18bfa0","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:44963","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":40,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:00.727468086Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:51604"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:00.727556027Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"36f5baaf-1d43-47e3-a3ad-f02623952cc3","remote_addr":"[fd00:1122:3344:101::c]:51604","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":97,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:00.73010419Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:42915"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:00.730289513Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"6b764507-8f39-4f7b-aaf6-b6964ae83111","remote_addr":"[fd00:1122:3344:101::c]:42915","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":111,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:00.734412894Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"3e51df5e-b4a0-465f-aab6-1c118c41d6fc","remote_addr":"[fd00:1122:3344:101::c]:51604","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":616,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:00.958346576Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:62403"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:00.958421397Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:00.998515156Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:01.058752176Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"990a6a47-8278-46ac-86f7-4b76a75fe1d1","remote_addr":"[fd00:1122:3344:101::c]:62403","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":100364,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:01.059040561Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"b474e5a8-6bc6-473c-8cc7-9d393837f0a0","remote_addr":"[fd00:1122:3344:101::c]:62403","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":110,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:01.299818838Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:48662"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:01.300070782Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"958a0e9a-dca8-48f1-8cfd-aa3ff7814fa8","remote_addr":"[fd00:1122:3344:101::b]:48662","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":199,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:01.31939143Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:58964"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:01.320210283Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"82b972fa-934f-4b66-a0be-9f36a867e4aa","remote_addr":"[fd00:1122:3344:101::b]:58964","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":794,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:01.328442693Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"6680e80a-7ced-41f9-bca5-6c27f0fed07a","remote_addr":"[fd00:1122:3344:101::b]:58964","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1031,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:01.341969472Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:63963"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:01.342096854Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"729e2ad0-8b61-403d-b3ae-3e51237648dc","remote_addr":"[fd00:1122:3344:101::a]:63963","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":89,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:01.423579814Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:53195"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:01.423705336Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"868c9797-5b0d-4f60-96d1-988cfc7f9ad1","remote_addr":"[fd00:1122:3344:101::a]:53195","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":110,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:01.429969122Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"7802cdb8-2d67-4475-b194-f6cfca0f11f9","remote_addr":"[fd00:1122:3344:101::a]:53195","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":99,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:01.515690784Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:56366"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:01.515782075Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:01.553562655Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:01.610973867Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"5705f140-2adb-48f9-bd37-91cfc4cbb1ce","remote_addr":"[fd00:1122:3344:101::b]:56366","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":95233,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:01.61111603Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:46712"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:01.611821752Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:01.648667966Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:01.710990541Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"224a3fb3-8802-4e42-8e6a-c1f10ce08597","remote_addr":"[fd00:1122:3344:101::a]:46712","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":99798,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:01.711331517Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"2e0f11a3-7e20-475d-972a-1a41c20d16e2","remote_addr":"[fd00:1122:3344:101::b]:56366","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":152,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:01.711462579Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"f5cd6740-eb98-497d-8056-69d1347cdb32","remote_addr":"[fd00:1122:3344:101::a]:46712","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":229,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:04.624118375Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:40808"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:04.624347159Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"204315c4-6c28-46e4-a247-600bb2ad1011","remote_addr":"[fd00:1122:3344:101::a]:40808","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":184,"response_code":"200"} {"msg":"Mapping virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:04.628260775Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:778","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 240, 227, 128])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(10829320) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:04.628292196Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"PUT","req_id":"56e2eb00-4eb8-4674-b477-51f3387139fb","remote_addr":"[fd00:1122:3344:101::a]:40808","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1730,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:04.771634593Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:54383"} {"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:04.783736898Z","hostname":"buskin","pid":1562,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:569","metadata":"InstanceMetadata { silo_id: 84f4ecf9-ccd6-4dde-85c8-c749363b2fc0, project_id: 75873103-03bb-4564-9380-cb8726446850 }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-08-23T22:04:04.266535Z }","instance_runtime":"InstanceRuntimeState { propolis_id: Some(1ddde876-90f4-44c8-a71b-91c98435b51a (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-08-23T22:04:04.327434450Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: ab4b9722-63b3-4972-8fe1-3031f72ea0c3, kind: Instance { id: 3c00b2c7-d4ec-4a69-acdb-190cdd1b0610 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 227, 128])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(10829320), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: ab4b9722-63b3-4972-8fe1-3031f72ea0c3, kind: Instance { id: 3c00b2c7-d4ec-4a69-acdb-190cdd1b0610 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 227, 128])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(10829320), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: ab4b9722-63b3-4972-8fe1-3031f72ea0c3, kind: Instance { id: 3c00b2c7-d4ec-4a69-acdb-190cdd1b0610 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 227, 128])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(10829320), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(10829320))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: ab4b9722-63b3-4972-8fe1-3031f72ea0c3, kind: Instance { id: 3c00b2c7-d4ec-4a69-acdb-190cdd1b0610 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 227, 128])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(10829320), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-491aa771766\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 5eaa0251-8bc1-48a1-8803-cb58a8bbfe9a, read_only_parent: Some(Volume { block_size: 512, id: 7524761e-59da-402c-a1b9-719097044aad, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 4712ad4e-9156-4dba-9c51-ba0326c965f5, key: Some(\"jW1c5RNZuQwtt44kNqIs0WM52ryD4P56RmdYsqD+pbI=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::1a]:19002\", \"[fd00:1122:3344:101::12]:19002\", \"[fd00:1122:3344:101::15]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 5eaa0251-8bc1-48a1-8803-cb58a8bbfe9a, key: Some(\"mMBh8eQ+6IHiSbQUR1SypnKTM3vaGv8fE2/Z6KzO1sU=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::13]:19001\", \"[fd00:1122:3344:101::14]:19000\", \"[fd00:1122:3344:101::16]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAABJgrAXWRdZAACCsBdZAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAASYKwF1kXWQAAgrAXWQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiIzYzAwYjJjNy1kNGVjLTRhNjktYWNkYi0xOTBjZGQxYjA2MTAiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJSnZMNE91amJUQnlkc0JLYkl6MjhDVmJBeUJKeTNNTTdqeG9NUHVQRVZZLyJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }","propolis_id":"1ddde876-90f4-44c8-a71b-91c98435b51a","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610"} {"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:04.78383482Z","hostname":"buskin","pid":1562,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:603","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610 (instance)"} {"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:04.783981292Z","hostname":"buskin","pid":1562,"instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","component":"InstanceManager","file":"sled-agent/src/instance.rs:994","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: ab4b9722-63b3-4972-8fe1-3031f72ea0c3, kind: Instance { id: 3c00b2c7-d4ec-4a69-acdb-190cdd1b0610 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 227, 128])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(10829320), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: ab4b9722-63b3-4972-8fe1-3031f72ea0c3, kind: Instance { id: 3c00b2c7-d4ec-4a69-acdb-190cdd1b0610 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 227, 128])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(10829320), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: ab4b9722-63b3-4972-8fe1-3031f72ea0c3, kind: Instance { id: 3c00b2c7-d4ec-4a69-acdb-190cdd1b0610 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 227, 128])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(10829320), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(10829320))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: ab4b9722-63b3-4972-8fe1-3031f72ea0c3, kind: Instance { id: 3c00b2c7-d4ec-4a69-acdb-190cdd1b0610 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 227, 128])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(10829320), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-491aa771766\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 5eaa0251-8bc1-48a1-8803-cb58a8bbfe9a, read_only_parent: Some(Volume { block_size: 512, id: 7524761e-59da-402c-a1b9-719097044aad, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 4712ad4e-9156-4dba-9c51-ba0326c965f5, key: Some(\"jW1c5RNZuQwtt44kNqIs0WM52ryD4P56RmdYsqD+pbI=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::1a]:19002\", \"[fd00:1122:3344:101::12]:19002\", \"[fd00:1122:3344:101::15]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 5eaa0251-8bc1-48a1-8803-cb58a8bbfe9a, key: Some(\"mMBh8eQ+6IHiSbQUR1SypnKTM3vaGv8fE2/Z6KzO1sU=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::13]:19001\", \"[fd00:1122:3344:101::14]:19000\", \"[fd00:1122:3344:101::16]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAABJgrAXWRdZAACCsBdZAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAASYKwF1kXWQAAgrAXWQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiIzYzAwYjJjNy1kNGVjLTRhNjktYWNkYi0xOTBjZGQxYjA2MTAiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJSnZMNE91amJUQnlkc0JLYkl6MjhDVmJBeUJKeTNNTTdqeG9NUHVQRVZZLyJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-08-23T22:04:04.266535Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400, migration_id: None }","migration_id":"None","propolis_id":"1ddde876-90f4-44c8-a71b-91c98435b51a","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:04.787558053Z","hostname":"buskin","pid":1562,"uri":"/instances/3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","method":"PUT","req_id":"db022aa3-ba50-4ddf-aba6-019a71183a0c","remote_addr":"[fd00:1122:3344:101::a]:54383","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":15860,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:04.924678795Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:50264"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:04.930456693Z","hostname":"buskin","pid":1562,"uri":"/instances/3c00b2c7-d4ec-4a69-acdb-190cdd1b0610/state","method":"PUT","req_id":"b6470628-dadc-4b76-8def-a0ec9c4297f2","remote_addr":"[fd00:1122:3344:101::a]:50264","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":5690,"response_code":"200"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:04.931764995Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:424","port":"Port { inner: PortInner(PortData { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 240, 227, 128]), slot: 0, vni: Vni { inner: 10829320 }, subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 } }) }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:04.932353595Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Configuring new Omicron zone: oxz_propolis-server_1ddde876-90f4-44c8-a71b-91c98435b51a","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:04.971300134Z","hostname":"buskin","pid":1562,"instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","component":"InstanceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_propolis-server_1ddde876-90f4-44c8-a71b-91c98435b51a","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:04.994526338Z","hostname":"buskin","pid":1562,"instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","component":"InstanceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_propolis-server_1ddde876-90f4-44c8-a71b-91c98435b51a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:06.527236723Z","hostname":"buskin","pid":1562,"instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_propolis-server_1ddde876-90f4-44c8-a71b-91c98435b51a zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:06.527513128Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_1ddde876-90f4-44c8-a71b-91c98435b51a","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_propolis-server_1ddde876-90f4-44c8-a71b-91c98435b51a\") failed: Property not found. retry in 31.986664ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T22:04:06.741496692Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_1ddde876-90f4-44c8-a71b-91c98435b51a","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_1ddde876-90f4-44c8-a71b-91c98435b51a\") failed: Property not found. retry in 63.609354ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T22:04:06.782747781Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_1ddde876-90f4-44c8-a71b-91c98435b51a","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_1ddde876-90f4-44c8-a71b-91c98435b51a\") failed: Property not found. retry in 262.020962ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T22:04:06.861924271Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_1ddde876-90f4-44c8-a71b-91c98435b51a","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_1ddde876-90f4-44c8-a71b-91c98435b51a\") failed: Property not found. retry in 424.740208ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T22:04:07.137298005Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_1ddde876-90f4-44c8-a71b-91c98435b51a","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_1ddde876-90f4-44c8-a71b-91c98435b51a\") failed: Property not found. retry in 1.104230603s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T22:04:07.578456226Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_1ddde876-90f4-44c8-a71b-91c98435b51a","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_1ddde876-90f4-44c8-a71b-91c98435b51a\") failed: Property not found. retry in 942.298785ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T22:04:08.695475352Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_1ddde876-90f4-44c8-a71b-91c98435b51a","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:09.400106205Z","hostname":"buskin","pid":1562,"uri":"/5050fe14-0845-40b2-872d-d70a5b2b56be","method":"GET","req_id":"672c1e59-fd29-4547-a38b-d8000cbad926","remote_addr":"[fd00:1122:3344:101::d]:62985","local_addr":"[fd00:1122:3344:101::1]:46625","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":11584,"response_code":"200"} {"msg":"wait for service Some(\"oxz_propolis-server_1ddde876-90f4-44c8-a71b-91c98435b51a\") failed: Property not found. retry in 623.998319ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T22:04:09.646844053Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_1ddde876-90f4-44c8-a71b-91c98435b51a","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_1ddde876-90f4-44c8-a71b-91c98435b51a\") failed: Property not found. retry in 555.162154ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T22:04:10.282790213Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_1ddde876-90f4-44c8-a71b-91c98435b51a","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_1ddde876-90f4-44c8-a71b-91c98435b51a\") failed: Property not found. retry in 1.489972482s","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T22:04:10.849713234Z","hostname":"buskin","pid":1562,"zone":"oxz_propolis-server_1ddde876-90f4-44c8-a71b-91c98435b51a","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Started propolis in zone: oxz_propolis-server_1ddde876-90f4-44c8-a71b-91c98435b51a","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:12.370222494Z","hostname":"buskin","pid":1562,"instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","component":"InstanceManager","file":"sled-agent/src/instance.rs:1448"} {"msg":"wait for service Some(\"oxz_propolis-server_1ddde876-90f4-44c8-a71b-91c98435b51a\") failed: Property not found. retry in 61.610866ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T22:04:12.38002859Z","hostname":"buskin","pid":1562,"instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_1ddde876-90f4-44c8-a71b-91c98435b51a\") failed: Property not found. retry in 122.755011ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T22:04:12.450986511Z","hostname":"buskin","pid":1562,"instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_1ddde876-90f4-44c8-a71b-91c98435b51a\") failed: Property not found. retry in 107.560314ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T22:04:12.582692232Z","hostname":"buskin","pid":1562,"instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_1ddde876-90f4-44c8-a71b-91c98435b51a\") failed: Property not found. retry in 207.738962ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T22:04:12.700659489Z","hostname":"buskin","pid":1562,"instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:12.919714989Z","hostname":"buskin","pid":1562,"instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","component":"InstanceManager","file":"sled-agent/src/instance.rs:1457"} {"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:12.975651727Z","hostname":"buskin","pid":1562,"instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","component":"InstanceManager","file":"sled-agent/src/instance.rs:1481"} {"msg":"Sending ensure request to propolis: InstanceEnsureRequest { cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAABJgrAXWRdZAACCsBdZAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAASYKwF1kXWQAAgrAXWQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiIzYzAwYjJjNy1kNGVjLTRhNjktYWNkYi0xOTBjZGQxYjA2MTAiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJSnZMNE91amJUQnlkc0JLYkl6MjhDVmJBeUJKeTNNTTdqeG9NUHVQRVZZLyJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"), disks: [DiskRequest { device: \"nvme\", name: \"disk-491aa771766\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 5eaa0251-8bc1-48a1-8803-cb58a8bbfe9a, read_only_parent: Some(Volume { block_size: 512, id: 7524761e-59da-402c-a1b9-719097044aad, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 4712ad4e-9156-4dba-9c51-ba0326c965f5, key: Some(\"jW1c5RNZuQwtt44kNqIs0WM52ryD4P56RmdYsqD+pbI=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::1a]:19002\", \"[fd00:1122:3344:101::12]:19002\", \"[fd00:1122:3344:101::15]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 5eaa0251-8bc1-48a1-8803-cb58a8bbfe9a, key: Some(\"mMBh8eQ+6IHiSbQUR1SypnKTM3vaGv8fE2/Z6KzO1sU=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::13]:19001\", \"[fd00:1122:3344:101::14]:19000\", \"[fd00:1122:3344:101::16]:19000\"] } }] } }], migrate: None, nics: [NetworkInterfaceRequest { name: \"opte6\", slot: Slot(0) }], properties: InstanceProperties { bootrom_id: 00000000-0000-0000-0000-000000000000, description: \"Test description\", id: 3c00b2c7-d4ec-4a69-acdb-190cdd1b0610, image_id: 00000000-0000-0000-0000-000000000000, memory: 1024, metadata: InstanceMetadata { project_id: 75873103-03bb-4564-9380-cb8726446850, silo_id: 84f4ecf9-ccd6-4dde-85c8-c749363b2fc0, sled_id: 5050fe14-0845-40b2-872d-d70a5b2b56be, sled_model: \"i86pc\", sled_revision: 0, sled_serial: \"buskin\" }, name: \"localshark\", vcpus: 2 } }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:12.97582894Z","hostname":"buskin","pid":1562,"instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","component":"InstanceManager","file":"sled-agent/src/instance.rs:730"} {"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:13.11816229Z","hostname":"buskin","pid":1562,"instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","component":"InstanceManager","file":"sled-agent/src/instance.rs:732"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Creating), migration_in: None, migration_out: None, time: 2024-08-23T22:04:13.120214555Z }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:13.120249685Z","hostname":"buskin","pid":1562,"instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","component":"InstanceManager","file":"sled-agent/src/instance.rs:600"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:13.120269686Z","hostname":"buskin","pid":1562,"instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","component":"InstanceManager","file":"sled-agent/src/instance.rs:622","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-08-23T22:04:13.120214555Z }","propolis_id":"1ddde876-90f4-44c8-a71b-91c98435b51a"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:13.120279486Z","hostname":"buskin","pid":1562,"instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","component":"InstanceManager","file":"sled-agent/src/instance.rs:520","state":"SledInstanceState { propolis_id: 1ddde876-90f4-44c8-a71b-91c98435b51a (propolis), vmm_state: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-08-23T22:04:13.120214555Z }, migration_in: None, migration_out: None }","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_in: None, migration_out: None, time: 2024-08-23T22:04:13.162041703Z }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:13.162064203Z","hostname":"buskin","pid":1562,"instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","component":"InstanceManager","file":"sled-agent/src/instance.rs:600"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:13.162080964Z","hostname":"buskin","pid":1562,"instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","component":"InstanceManager","file":"sled-agent/src/instance.rs:622","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-08-23T22:04:13.162041703Z }","propolis_id":"1ddde876-90f4-44c8-a71b-91c98435b51a"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:13.162088554Z","hostname":"buskin","pid":1562,"instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","component":"InstanceManager","file":"sled-agent/src/instance.rs:520","state":"SledInstanceState { propolis_id: 1ddde876-90f4-44c8-a71b-91c98435b51a (propolis), vmm_state: VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-08-23T22:04:13.162041703Z }, migration_in: None, migration_out: None }","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:26.539777862Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36302"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:26.539858513Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"bd9a4a70-677c-4eac-a0d4-5f3828585600","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36302","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":58,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:30.814176698Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:58600"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:30.8142547Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"6c5d5915-3a65-4b63-9898-fd7c2141fae9","remote_addr":"[fd00:1122:3344:101::c]:58600","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":69,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:30.859814281Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"ba3ea461-6a71-4d9e-92bb-4a219cf15da4","remote_addr":"[fd00:1122:3344:101::c]:58600","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":853,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:30.953837454Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:33572"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:30.956514559Z","hostname":"buskin","pid":1562,"uri":"/instances/3c00b2c7-d4ec-4a69-acdb-190cdd1b0610/state","method":"GET","req_id":"4e689a84-4740-4439-8448-d5c1cf238da9","remote_addr":"[fd00:1122:3344:101::c]:33572","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":117,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:31.079840688Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:52090"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:31.080713982Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"76d9357a-10fd-4895-963b-adf6cea5e756","remote_addr":"[fd00:1122:3344:101::c]:52090","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":844,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:31.240121602Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:58253"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:31.240269094Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:31.279084482Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:31.345184871Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"499ee77f-ba1d-4bff-95fb-cdf202fcf138","remote_addr":"[fd00:1122:3344:101::c]:58253","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":104987,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:31.345335373Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:43793"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:31.345618388Z","hostname":"buskin","pid":1562,"uri":"/instances/3c00b2c7-d4ec-4a69-acdb-190cdd1b0610/state","method":"GET","req_id":"e2b9ad0c-5d01-4701-8300-df246f391214","remote_addr":"[fd00:1122:3344:101::b]:43793","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":212,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:31.346696467Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"04e125e5-36bc-43e1-9842-44616a79d0db","remote_addr":"[fd00:1122:3344:101::c]:58253","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1207,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:31.393459219Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:48591"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:31.394660349Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"3b744075-1d7f-4f26-abc2-5bbce130a773","remote_addr":"[fd00:1122:3344:101::a]:48591","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1145,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:31.405581744Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:58828"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:31.405682766Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"39632505-b85c-4419-8a58-34f0d7a779dd","remote_addr":"[fd00:1122:3344:101::b]:58828","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":66,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:31.419599551Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"9ee725cc-9b7c-43a4-b123-5b7ca0c49d63","remote_addr":"[fd00:1122:3344:101::b]:58828","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":143,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:31.449113581Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:56346"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:31.449250213Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"3be106a9-23b5-41ba-8d73-1b55673dd740","remote_addr":"[fd00:1122:3344:101::b]:56346","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":108,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:31.500709355Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:45397"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:31.500792486Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"b6d0852e-1656-4f8e-8a03-5fa061009434","remote_addr":"[fd00:1122:3344:101::a]:45397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":52,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:31.512178329Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"d4d2098a-da75-4224-91df-106215ff98b4","remote_addr":"[fd00:1122:3344:101::a]:45397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":90,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:31.531044269Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:51788"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:31.533646063Z","hostname":"buskin","pid":1562,"uri":"/instances/3c00b2c7-d4ec-4a69-acdb-190cdd1b0610/state","method":"GET","req_id":"f6b9f601-b724-4fe6-b94d-8ab37fe8757a","remote_addr":"[fd00:1122:3344:101::a]:51788","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":82,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:31.744622366Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:43515"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:31.744720767Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:31.783673107Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:31.840600941Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"dccd3984-891c-4fa3-8c9e-b3150600d5e9","remote_addr":"[fd00:1122:3344:101::b]:43515","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":95931,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:31.840747623Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:43209"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:31.840921446Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4033"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:31.844635389Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"a1d29a18-b48f-42af-8907-3399cca3fb4b","remote_addr":"[fd00:1122:3344:101::b]:43515","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":3649,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:31.875185297Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:31.930270459Z","hostname":"buskin","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"6a614e8e-9629-4c1d-8b5f-794d66482be0","remote_addr":"[fd00:1122:3344:101::a]:43209","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":89434,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:31.930615545Z","hostname":"buskin","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"a32e3f2d-83a7-4a0c-90a9-2b42c6af09d0","remote_addr":"[fd00:1122:3344:101::a]:43209","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":154,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:38.594526547Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:59349"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:38.595101897Z","hostname":"buskin","pid":1562,"uri":"/instances/3c00b2c7-d4ec-4a69-acdb-190cdd1b0610/state","method":"PUT","req_id":"e573b585-749e-4531-b184-7ac4d5c24e9e","remote_addr":"[fd00:1122:3344:101::a]:59349","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":527,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_in: None, migration_out: None, time: 2024-08-23T22:04:38.595241149Z }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:38.59526719Z","hostname":"buskin","pid":1562,"instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","component":"InstanceManager","file":"sled-agent/src/instance.rs:600"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:38.59527882Z","hostname":"buskin","pid":1562,"instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","component":"InstanceManager","file":"sled-agent/src/instance.rs:622","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-08-23T22:04:38.595241149Z }","propolis_id":"1ddde876-90f4-44c8-a71b-91c98435b51a"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:38.595308561Z","hostname":"buskin","pid":1562,"instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","component":"InstanceManager","file":"sled-agent/src/instance.rs:520","state":"SledInstanceState { propolis_id: 1ddde876-90f4-44c8-a71b-91c98435b51a (propolis), vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-08-23T22:04:38.595241149Z }, migration_in: None, migration_out: None }","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_in: None, migration_out: None, time: 2024-08-23T22:04:38.632015802Z }","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:38.632037123Z","hostname":"buskin","pid":1562,"instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","component":"InstanceManager","file":"sled-agent/src/instance.rs:600"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:38.632068683Z","hostname":"buskin","pid":1562,"instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","component":"InstanceManager","file":"sled-agent/src/instance.rs:622","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(6), time_updated: 2024-08-23T22:04:38.632015802Z }","propolis_id":"1ddde876-90f4-44c8-a71b-91c98435b51a"} {"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:38.632098504Z","hostname":"buskin","pid":1562,"instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","component":"InstanceManager","file":"sled-agent/src/instance.rs:636","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:38.632106874Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"target removed with queued samples","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T22:04:38.632199415Z","hostname":"buskin","pid":1562,"component":"kstat-sampler-worker","sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:557","n_samples":12,"id":"12316762901450333873"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:38.632406229Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:38.632516791Z","hostname":"buskin","pid":1562,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:339","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/156808b0-3956-455e-a8e1-2217e0afca54/debug/bundle/zone\", \"/pool/int/fe7217f3-1112-4ad5-a4fc-b58965332aab/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/c4828395-89b1-4653-93a7-55b242871c48/crypt/debug\", \"/pool/ext/c4fe4e01-40de-4c9e-857a-e6870bd84153/crypt/debug\", \"/pool/ext/0bb3d35c-cbec-445b-9b3a-58337934f8de/crypt/debug\", \"/pool/ext/44b92f88-0c07-4732-953e-9cac63715495/crypt/debug\", \"/pool/ext/44ff68f6-909f-4df8-8e90-d68ca84323b3/crypt/debug\", \"/pool/ext/dad70eae-b83f-4c3a-a32e-2a6a8946803c/crypt/debug\", \"/pool/ext/ec50006e-a21c-4820-b439-48f6d6b60c97/crypt/debug\", \"/pool/ext/e1a016b1-16ad-48d4-a26f-2e916f51bc80/crypt/debug\", \"/pool/ext/7ac03450-9a27-4ec4-9669-9eb798da4419/crypt/debug\"] }","zone_name":"oxz_propolis-server_1ddde876-90f4-44c8-a71b-91c98435b51a"} {"msg":"target removed with queued samples","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T22:04:38.632688983Z","hostname":"buskin","pid":1562,"component":"kstat-sampler-worker","sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:557","n_samples":12,"id":"13157610138689833664"} {"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:39.2543073Z","hostname":"buskin","pid":1562,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1137","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_1ddde876-90f4-44c8-a71b-91c98435b51a\", bundle_id: 004b52aa-c9b0-4370-ab42-1a7ca67e68c9 }, time_created: 2024-08-23T22:04:38.633119591Z, version: 0, cause: TerminatedInstance }"} {"msg":"Halting and removing zone: oxz_propolis-server_1ddde876-90f4-44c8-a71b-91c98435b51a","v":0,"name":"SledAgent","level":40,"time":"2024-08-23T22:04:39.254396762Z","hostname":"buskin","pid":1562,"instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","component":"InstanceManager","file":"sled-agent/src/instance.rs:831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:39.395789326Z","hostname":"buskin","pid":1562,"uri":"/5050fe14-0845-40b2-872d-d70a5b2b56be","method":"GET","req_id":"ddb66766-aa50-46c0-9214-1f5c29ea5db4","remote_addr":"[fd00:1122:3344:101::d]:62985","local_addr":"[fd00:1122:3344:101::1]:46625","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","component":"producer-server","sled_id":"5050fe14-0845-40b2-872d-d70a5b2b56be","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":7598,"response_code":"200"} {"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:41.00511816Z","hostname":"buskin","pid":1562,"instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","component":"InstanceManager","file":"illumos-utils/src/zone.rs:270"} {"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:41.021552379Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:904","id":"RouterId { vni: Vni(10829320), kind: System }"} {"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:41.02160357Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:904","id":"RouterId { vni: Vni(10829320), kind: Custom(V4(Ipv4Net { addr: 172.30.0.0, width: 22 })) }"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:41.022136519Z","hostname":"buskin","pid":1562,"instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","component":"InstanceManager","file":"sled-agent/src/instance.rs:520","state":"SledInstanceState { propolis_id: 1ddde876-90f4-44c8-a71b-91c98435b51a (propolis), vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-08-23T22:04:41.022105498Z }, migration_in: None, migration_out: None }","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:41.049326169Z","hostname":"buskin","pid":1562,"instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","component":"InstanceManager","file":"sled-agent/src/instance.rs:520","state":"SledInstanceState { propolis_id: 1ddde876-90f4-44c8-a71b-91c98435b51a (propolis), vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-08-23T22:04:41.022105498Z }, migration_in: None, migration_out: None }","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610"} {"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:41.054496327Z","hostname":"buskin","pid":1562,"instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","instance_id":"3c00b2c7-d4ec-4a69-acdb-190cdd1b0610","component":"InstanceManager","file":"sled-agent/src/instance.rs:768"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:41.434222337Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:44729"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:41.434331199Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"898a3775-2671-4f27-ae98-48b5085d4d83","remote_addr":"[fd00:1122:3344:101::a]:44729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":102,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:41.439519347Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"01c35d97-cfe7-48e1-b9eb-c6b3b5ae3a54","remote_addr":"[fd00:1122:3344:101::a]:44729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":126,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:41.45211761Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:50439"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:41.452798092Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"ab79807d-de0c-4232-b3e9-6ca924f3d385","remote_addr":"[fd00:1122:3344:101::a]:50439","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":664,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:41.730151729Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:36468"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:41.730324921Z","hostname":"buskin","pid":1562,"uri":"/v2p","method":"GET","req_id":"1a27c674-443b-46ba-94c4-2d56898a3876","remote_addr":"[fd00:1122:3344:101::a]:36468","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":152,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:42.764527835Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:63700"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:42.764629157Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"GET","req_id":"69975a7f-2853-4227-86b0-b58a957c446f","remote_addr":"[fd00:1122:3344:101::a]:63700","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":106,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-23T22:04:42.770510297Z","hostname":"buskin","pid":1562,"uri":"/vpc-routes","method":"PUT","req_id":"c4d3fe51-73fc-4a4d-a96f-3c739708254a","remote_addr":"[fd00:1122:3344:101::a]:63700","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":102,"response_code":"204"}