[ Jul 10 23:53:16 Enabled. ]
[ Jul 10 23:53:16 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ]
[ Jul 10 23:53:16 Method "start" exited with status 0. ]
[ Jul 10 23:53:16 Rereading configuration. ]
[ Jul 10 23:53:16 No 'refresh' method defined.  Treating as :true. ]
note: configured to log to "/dev/stdout"
{"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:16.301637034Z","hostname":"buskin","pid":1555,"file":"illumos-utils/src/running_zone.rs:163"}
{"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:16.301813976Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"}
{"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:16.35639876Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"}
{"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:16.35644038Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"}
{"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:16.472347851Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"}
{"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:16.481782323Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"}
{"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:16.489882781Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.093187431Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fdb0:18c0:4d0d:9fb2::, width: 64 }"}
{"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.233240634Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:119"}
{"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.233280064Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:131"}
{"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.233382495Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:141"}
{"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.233467406Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:162","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"}
{"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.233494666Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:506"}
{"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.233514306Z","hostname":"buskin","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:668"}
{"msg":"Generated i86pc baseboard Pc { identifier: \"buskin\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.236662291Z","hostname":"buskin","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:575"}
{"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:17.236670091Z","hostname":"buskin","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:717"}
{"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.236708441Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:176"}
{"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.236720921Z","hostname":"buskin","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:776"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.236726612Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/m2_0.vdev"}
{"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.236793842Z","hostname":"buskin","pid":1555,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"None"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.236900383Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.236927244Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:495","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.236947634Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.248076175Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"}
{"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.248115135Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:372","id":"cac444c1-8feb-42ab-bee7-aa35319fb578 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.348225234Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: cac444c1-8feb-42ab-bee7-aa35319fb578 (zpool), kind: Internal }"}
{"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.546129826Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.546163546Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/m2_1.vdev"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.546168636Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:564","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.546182937Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:585","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.546190026Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.546214557Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:495","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.546219527Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.554592378Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"}
{"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.554619078Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:372","id":"c0d1014c-d647-4991-8139-bcd86ea27898 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.640687924Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: c0d1014c-d647-4991-8139-bcd86ea27898 (zpool), kind: Internal }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.843601961Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_0.vdev"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.843644081Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: cac444c1-8feb-42ab-bee7-aa35319fb578 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: c0d1014c-d647-4991-8139-bcd86ea27898 (zpool), kind: Internal } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.843685601Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.843692452Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.843698022Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.843705382Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:495","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.843710682Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_1.vdev"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.843715152Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: cac444c1-8feb-42ab-bee7-aa35319fb578 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: c0d1014c-d647-4991-8139-bcd86ea27898 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.843727832Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.843737982Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.843743282Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.843758802Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:495","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.843764162Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_2.vdev"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.843768422Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: cac444c1-8feb-42ab-bee7-aa35319fb578 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: c0d1014c-d647-4991-8139-bcd86ea27898 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.843785023Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.843794083Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.843799323Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.843804783Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:495","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.843809903Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_3.vdev"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.843815183Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.843822053Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: cac444c1-8feb-42ab-bee7-aa35319fb578 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: c0d1014c-d647-4991-8139-bcd86ea27898 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.843847563Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:495","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.843852823Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.843857854Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.843861723Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_4.vdev"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.843866774Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: cac444c1-8feb-42ab-bee7-aa35319fb578 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: c0d1014c-d647-4991-8139-bcd86ea27898 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.843886744Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.843891964Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.843896624Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.843903164Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:495","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.843908104Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_5.vdev"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.843913124Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: cac444c1-8feb-42ab-bee7-aa35319fb578 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: c0d1014c-d647-4991-8139-bcd86ea27898 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.843942764Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.843949175Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.843954395Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.843958795Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:495","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.843968345Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_6.vdev"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.843973495Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.843977655Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:495","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.843983745Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: cac444c1-8feb-42ab-bee7-aa35319fb578 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: c0d1014c-d647-4991-8139-bcd86ea27898 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.844015065Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_7.vdev"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.844019945Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.844025125Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:495","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.844029435Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: cac444c1-8feb-42ab-bee7-aa35319fb578 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: c0d1014c-d647-4991-8139-bcd86ea27898 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.844051165Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.844055445Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.844058506Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.844075446Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.844079176Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_8.vdev"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.844085336Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.844089266Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:495","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.844094966Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: cac444c1-8feb-42ab-bee7-aa35319fb578 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: c0d1014c-d647-4991-8139-bcd86ea27898 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.844121576Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:91"}
{"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.844130686Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:93"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.844134366Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.844137626Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: cac444c1-8feb-42ab-bee7-aa35319fb578 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: c0d1014c-d647-4991-8139-bcd86ea27898 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.844166797Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.844171117Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.844174527Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.844178727Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"}
{"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.844182057Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:200"}
{"msg":"No ledger in /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.844185757Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.844193127Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"}
{"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.844197357Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"}
{"msg":"No ledger in /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.844203147Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.844206937Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"}
{"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.844210747Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"}
{"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.844214277Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:205"}
{"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.844217567Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:220"}
{"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.85454744Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:712"}
{"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.85459141Z","hostname":"buskin","pid":1555,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:289","time_to_next_cleanup":"599.999919979s","next_cleanup":"Instant { tv_sec: 1620, tv_nsec: 968418760 }"}
{"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.854651751Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3568"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.854658901Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"No ledger in /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.854716301Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.857239599Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"}
{"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.857650773Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:196"}
{"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.857716144Z","hostname":"buskin","pid":1555,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:44","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:12346"}
{"msg":"Injecting RSS configuration: RackInitializeRequest {\n    trust_quorum_peers: None,\n    bootstrap_discovery: OnlyOurs,\n    ntp_servers: [\n        \"0.pool.ntp.org\",\n    ],\n    dns_servers: [\n        1.1.1.1,\n        9.9.9.9,\n    ],\n    internal_services_ip_pool_ranges: [\n        V4(\n            Ipv4Range {\n                first: 10.151.1.164,\n                last: 10.151.1.173,\n            },\n        ),\n    ],\n    external_dns_ips: [\n        10.151.1.164,\n        10.151.1.165,\n    ],\n    external_dns_zone_name: \"oxide.test\",\n    external_certificates: \"<redacted>\",\n    recovery_silo: RecoverySiloConfig {\n        silo_name: Name(\n            \"recovery\",\n        ),\n        user_name: UserId(\n            \"recovery\",\n        ),\n        user_password_hash: NewPasswordHash(\n            PasswordHashString {\n                string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n                encoding: B64,\n            },\n        ),\n    },\n    rack_network_config: RackNetworkConfigV2 {\n        rack_subnet: Ipv6Net {\n            addr: fd00:1122:3344:100::,\n            width: 56,\n        },\n        infra_ip_first: 10.151.1.174,\n        infra_ip_last: 10.151.1.174,\n        ports: [\n            PortConfigV2 {\n                routes: [\n                    RouteConfig {\n                        destination: V4(\n                            Ipv4Net {\n                                addr: 0.0.0.0,\n                                width: 0,\n                            },\n                        ),\n                        nexthop: 10.151.1.100,\n                        vlan_id: None,\n                    },\n                ],\n                addresses: [\n                    UplinkAddressConfig {\n                        address: V4(\n                            Ipv4Net {\n                                addr: 10.151.1.174,\n                                width: 24,\n                            },\n                        ),\n                        vlan_id: None,\n                    },\n                ],\n                switch: Switch0,\n                port: \"qsfp0\",\n                uplink_port_speed: Speed40G,\n                uplink_port_fec: None,\n                bgp_peers: [],\n                autoneg: false,\n            },\n        ],\n        bgp: [],\n        bfd: [],\n    },\n    allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.857922126Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1011"}
{"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.857928567Z","hostname":"buskin","pid":1555,"component":"DnsResolver","component":"RSS","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.857963317Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"No ledger in /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.858016227Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.858025088Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.858031728Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1071"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.858048458Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"No ledger in /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.858060448Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.858066458Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.858071018Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1112"}
{"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.858074898Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:156"}
{"msg":"Creating plan for the sled at fdb0:18c0:4d0d:9fb2::1","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.858079078Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:129"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.858112358Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Writing ledger to /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.858141529Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.858748055Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Sled plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.858934037Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:176"}
{"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.858941038Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1145"}
{"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.858981118Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"}
{"msg":"Writing ledger to /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.858987168Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.85913452Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"}
{"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.859258291Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"}
{"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.859295611Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:217","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 0dcbb472-7cd1-4e42-9116-fc1cc27da573, rack_id: 42fd4f23-0a70-4234-8dfb-155c06b26fe7, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }"}
{"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.859387392Z","hostname":"buskin","pid":1555,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:34569","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:66"}
{"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 0dcbb472-7cd1-4e42-9116-fc1cc27da573, rack_id: 42fd4f23-0a70-4234-8dfb-155c06b26fe7, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.859530854Z","hostname":"buskin","pid":1555,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:360"}
{"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.859541084Z","hostname":"buskin","pid":1555,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:375"}
{"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.859570624Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.859577094Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:574"}
{"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.859580384Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/server.rs:44"}
{"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.859601835Z","hostname":"buskin","pid":1555,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.859606395Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:101::, width: 64 }"}
{"msg":"No ledger in /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.859611885Z","hostname":"buskin","pid":1555,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.859616655Z","hostname":"buskin","pid":1555,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"}
{"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.859625725Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:563"}
{"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.859629405Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:591"}
{"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.862544797Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3800"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.869441112Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:647","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.878201937Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:647","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.887965363Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:647","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.896705438Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:647","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.90515339Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:647","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.913603292Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:647","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.921883622Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:647","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.930074441Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:647","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.938218209Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:647","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"}
{"msg":"No disks to be automatically managed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.93826471Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:602"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.93827045Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.93827672Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:395"}
{"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.93831875Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:340"}
{"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.938324151Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:481","result":"DisksManagementResult { status: [] }"}
{"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.958672362Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.975272563Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.998137841Z","hostname":"buskin","pid":1555,"sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:355"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.998168921Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.998192762Z","hostname":"buskin","pid":1555,"sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:367"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:17.998200412Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 138.540977ms","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:18.001421257Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b247b628-71ee-42c3-b0cd-63af1b4e6644\", \"content-length\": \"124\", \"date\": \"Wed, 10 Jul 2024 23:53:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b247b628-71ee-42c3-b0cd-63af1b4e6644\" }"}
{"msg":"attempting to create encrypted zvol: name=\"oxi_cac444c1-8feb-42ab-bee7-aa35319fb578/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:18.011218313Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/swap_device.rs:153"}
{"msg":"successfully created encrypted zvol: name=\"oxi_cac444c1-8feb-42ab-bee7-aa35319fb578/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:18.03663742Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/swap_device.rs:249"}
{"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_cac444c1-8feb-42ab-bee7-aa35319fb578/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:18.03667894Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/swap_device.rs:96"}
{"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:18.042032628Z","hostname":"buskin","pid":1555,"sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:382"}
{"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:18.042040978Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:126"}
{"msg":"Ensuring dataset oxi_cac444c1-8feb-42ab-bee7-aa35319fb578/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:18.042045178Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:136"}
{"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:18.116805011Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:170"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:18.140925694Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 105.980886ms","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:18.141556921Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c0b264a0-d20c-4780-a972-5f8bede2394e\", \"content-length\": \"124\", \"date\": \"Wed, 10 Jul 2024 23:53:18 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c0b264a0-d20c-4780-a972-5f8bede2394e\" }"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:18.247638784Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 121.161059ms","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:18.248063359Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d2756d1a-78f0-408d-98e1-1156bd8db264\", \"content-length\": \"124\", \"date\": \"Wed, 10 Jul 2024 23:53:18 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d2756d1a-78f0-408d-98e1-1156bd8db264\" }"}
{"msg":"Mounting oxi_cac444c1-8feb-42ab-bee7-aa35319fb578/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:18.33551701Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:178"}
{"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:18.349257299Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"}
{"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:18.34932856Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"}
{"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:18.34934303Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"}
{"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:18.34937331Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:192"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:18.370353619Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 242.810012ms","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:18.370880704Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e25f6458-525c-4c91-bcc7-df6d9cca9d71\", \"content-length\": \"124\", \"date\": \"Wed, 10 Jul 2024 23:53:18 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e25f6458-525c-4c91-bcc7-df6d9cca9d71\" }"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:18.615403123Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 299.495592ms","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:18.616185802Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9635dabc-e25d-4079-932d-06ec11590046\", \"content-length\": \"124\", \"date\": \"Wed, 10 Jul 2024 23:53:18 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9635dabc-e25d-4079-932d-06ec11590046\" }"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:18.917307506Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 162.589827ms","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:18.917801321Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e398b18c-5edc-4c54-b3d2-5b4c524f1354\", \"content-length\": \"124\", \"date\": \"Wed, 10 Jul 2024 23:53:18 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e398b18c-5edc-4c54-b3d2-5b4c524f1354\" }"}
{"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:19.077151464Z","hostname":"buskin","pid":1555,"sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:90"}
{"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:19.077763091Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:63316","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:196","component":"producer-server","component":"MetricsManager","sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:196"}
{"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:19.077784931Z","hostname":"buskin","pid":1555,"component":"internal-dns-resolver","file":"internal-dns/src/resolver.rs:60","component":"producer-server","component":"MetricsManager","sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"}
{"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:19.077844292Z","hostname":"buskin","pid":1555,"file":"oximeter/producer/src/lib.rs:282","component":"producer-server","component":"MetricsManager","sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"oximeter/producer/src/lib.rs:282","interval":"30s","address":"[fd00:1122:3344:101::1]:63316","producer_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573"}
{"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:19.078987324Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:390","component":"producer-server","component":"MetricsManager","sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"oximeter/producer/src/lib.rs:390","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"301.520772ms"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:19.08133369Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:19.125172567Z","hostname":"buskin","pid":1555,"component":"VmmReservoirManager","sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"}
{"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:19.125200507Z","hostname":"buskin","pid":1555,"component":"VmmReservoirManager","sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"}
{"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:19.125346988Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:895","underlay_address":"fd00:1122:3344:101::1"}
{"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:19.125366338Z","hostname":"buskin","pid":1555,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"}
{"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:19.126246418Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:196"}
{"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:19.126283468Z","hostname":"buskin","pid":1555,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:430"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:19.126309979Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Writing ledger to /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:19.126329599Z","hostname":"buskin","pid":1555,"component":"SledAgentMain","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:19.126587962Z","hostname":"buskin","pid":1555,"component":"SledAgentMain","file":"common/src/ledger.rs:199"}
{"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:19.126770824Z","hostname":"buskin","pid":1555,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"}
{"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:19.126801164Z","hostname":"buskin","pid":1555,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV2 { rack_subnet: Ipv6Net { addr: fd00:1122:3344:100::, width: 56 }, infra_ip_first: 10.151.1.174, infra_ip_last: 10.151.1.174, ports: [PortConfigV2 { routes: [RouteConfig { destination: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), nexthop: 10.151.1.100, vlan_id: None }], addresses: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false }], bgp: [], bfd: [] })))"}
{"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:19.126815324Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3568"}
{"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:19.126822264Z","hostname":"buskin","pid":1555,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:34569","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:73"}
{"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:19.126875405Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:108","peer":"[fdb0:18c0:4d0d:9fb2::1]:12346"}
{"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:19.126890015Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:19.304669028Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"No ledger in /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:19.304754659Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:19.304766289Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:19.304772189Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:19.30477774Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/0dcbb472-7cd1-4e42-9116-fc1cc27da573): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:19.310004416Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:19.336230751Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:42891"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:19.336390673Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:19.336485394Z","hostname":"buskin","pid":1555,"uri":"/inventory","method":"GET","req_id":"94f9027b-2377-4af1-8de6-cc53c834a93f","remote_addr":"[fd00:1122:3344:101::1]:42891","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":162,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:19.365010475Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:59215"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:19.365140376Z","hostname":"buskin","pid":1555,"uri":"/sled-role","method":"GET","req_id":"b2d870e0-1549-4774-a6e4-3e831bac3a30","remote_addr":"[fd00:1122:3344:101::1]:59215","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":60,"response_code":"200"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:19.366619992Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Writing ledger to /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:19.366647562Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:19.366960106Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:19.367182158Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:832"}
{"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:19.381993529Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:390","component":"producer-server","component":"MetricsManager","sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"oximeter/producer/src/lib.rs:390","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"267.995896ms"}
{"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { disks: [OmicronPhysicalDiskConfig { id: 03e850c1-0434-4105-b098-852bbd0c4170, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, pool_id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool) }, OmicronPhysicalDiskConfig { id: 39b16867-3a0c-4ffd-93a2-f4dcf8414528, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, pool_id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool) }, OmicronPhysicalDiskConfig { id: 5e60e5fe-7465-4003-885b-2b346bf2d85a, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, pool_id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool) }, OmicronPhysicalDiskConfig { id: 9271791b-5e74-4ecf-8ba2-a93f1c594d08, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, pool_id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool) }, OmicronPhysicalDiskConfig { id: 9dd8edbe-7897-4661-8689-9a40925a8373, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, pool_id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool) }, OmicronPhysicalDiskConfig { id: d2f2e42e-85ae-46cc-8aa2-b9f7d74af543, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, pool_id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool) }, OmicronPhysicalDiskConfig { id: ac83b964-d05c-4b4c-8466-c2c86034a6a2, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, pool_id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool) }, OmicronPhysicalDiskConfig { id: 8796c2d6-ad7b-4b67-83e4-133c102eb13c, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, pool_id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool) }, OmicronPhysicalDiskConfig { id: acf36bba-d120-406c-b6b8-aa7b22ac6df3, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, pool_id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool) }], generation: Generation(1) }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:19.397590059Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:348"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:19.39768141Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:56903"}
{"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 03e850c1-0434-4105-b098-852bbd0c4170, pool_id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: 39b16867-3a0c-4ffd-93a2-f4dcf8414528, pool_id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: 5e60e5fe-7465-4003-885b-2b346bf2d85a, pool_id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 9271791b-5e74-4ecf-8ba2-a93f1c594d08, pool_id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 9dd8edbe-7897-4661-8689-9a40925a8373, pool_id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: d2f2e42e-85ae-46cc-8aa2-b9f7d74af543, pool_id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: ac83b964-d05c-4b4c-8466-c2c86034a6a2, pool_id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 8796c2d6-ad7b-4b67-83e4-133c102eb13c, pool_id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: acf36bba-d120-406c-b6b8-aa7b22ac6df3, pool_id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:19.397940293Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"No ledger in /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:19.397946173Z","hostname":"buskin","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:19.397951763Z","hostname":"buskin","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"}
{"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:19.397956083Z","hostname":"buskin","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:725"}
{"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:19.397961793Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:340"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:19.397966303Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:380","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:19.397971673Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:457"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:19.397975953Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:19.496274412Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:19.496329392Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"2ff702b9-0076-4c58-8af6-039be08bef86 (zpool)"}
{"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:19.675772584Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:390","component":"producer-server","component":"MetricsManager","sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"oximeter/producer/src/lib.rs:390","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"991.806129ms"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/0dcbb472-7cd1-4e42-9116-fc1cc27da573): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-10T23:53:20.127705358Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:171"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:20.482974031Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:20.652465444Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:20.652534775Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:20.652544065Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:20.652559995Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:20.652568995Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:20.652717827Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:20.652744567Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:20.885781871Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:390","component":"producer-server","component":"MetricsManager","sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"oximeter/producer/src/lib.rs:390","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"1.059048992s"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:21.268999298Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:21.472022276Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/0dcbb472-7cd1-4e42-9116-fc1cc27da573): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-10T23:53:21.666562171Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:171"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/0dcbb472-7cd1-4e42-9116-fc1cc27da573): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:21.666831214Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"Automatically destroying dataset: oxp_2ff702b9-0076-4c58-8af6-039be08bef86/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:21.89080822Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 30.407111ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:22.071971399Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:22.073063771Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:390","component":"producer-server","component":"MetricsManager","sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"oximeter/producer/src/lib.rs:390","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"5.107341407s"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 134.309401ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:22.599498465Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/0dcbb472-7cd1-4e42-9116-fc1cc27da573): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-10T23:53:23.217874479Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:171"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 126.691122ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:23.218306554Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/0dcbb472-7cd1-4e42-9116-fc1cc27da573): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:23.729451512Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 437.820621ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:23.729817586Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 896.51111ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:24.645548514Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/0dcbb472-7cd1-4e42-9116-fc1cc27da573): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-10T23:53:24.646497704Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:171"}
{"msg":"Looking for unencrypted datasets in oxp_2ff702b9-0076-4c58-8af6-039be08bef86","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:24.663316617Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:24.915726822Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:478","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:24.915776682Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:395","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:24.915783162Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:380","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:24.915790833Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:457"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:24.915795353Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:25.156440529Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:25.156471599Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:26.31621419Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:26.561937662Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:26.561978452Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:26.561986072Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:26.562010763Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:26.562019853Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:26.562198565Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:26.562207355Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/0dcbb472-7cd1-4e42-9116-fc1cc27da573): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:27.065425216Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/0dcbb472-7cd1-4e42-9116-fc1cc27da573): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-10T23:53:27.53476803Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:171"}
{"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:27.535104034Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:390","component":"producer-server","component":"MetricsManager","sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"oximeter/producer/src/lib.rs:390","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"11.129569214s"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:28.107491557Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_960eb39e-b913-4eb7-b0fb-372e6f629bcb/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:28.619745648Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/0dcbb472-7cd1-4e42-9116-fc1cc27da573): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-10T23:53:29.338278381Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:171"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/0dcbb472-7cd1-4e42-9116-fc1cc27da573): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:29.338325961Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:29.556221251Z","hostname":"buskin","pid":1555,"component":"VmmReservoirManager","sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"}
{"msg":"Looking for unencrypted datasets in oxp_960eb39e-b913-4eb7-b0fb-372e6f629bcb","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:29.664914222Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:29.672865849Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:478","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:29.67291916Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:395","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:29.6729301Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:380","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:29.67293889Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:457"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:29.67296669Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:29.682540414Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:29.682588005Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:29.776011181Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","zpool":"ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:29.782865995Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:29.782909456Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:29.782920056Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:29.782932766Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:29.782996586Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:29.783146678Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:29.783156678Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:29.818127459Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_f91400c1-df8c-4cdb-b4f6-cb47c0d105a8/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:29.826415589Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"Looking for unencrypted datasets in oxp_f91400c1-df8c-4cdb-b4f6-cb47c0d105a8","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:29.954217968Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:29.9626212Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:478","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:29.96266996Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:395","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:29.96267691Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:380","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:29.96268713Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:457"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:29.96269252Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:29.972941282Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:29.972983972Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.070769846Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","zpool":"ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.078621041Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.078667412Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.078675222Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.078686812Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.078712402Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.078835103Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.078842134Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.113119556Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_d29abc28-1ae7-4192-8340-9766fa8fec22/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.120751689Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"Looking for unencrypted datasets in oxp_d29abc28-1ae7-4192-8340-9766fa8fec22","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.243533854Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.251345729Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:478","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.25138131Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:395","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.25138685Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:380","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.25139511Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:457"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.25139966Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.261073835Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.261110156Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.356417292Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","zpool":"ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.364386848Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.364414819Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.364422319Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.364444409Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.364453479Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.364625781Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.364635841Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/0dcbb472-7cd1-4e42-9116-fc1cc27da573): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-10T23:53:30.368765816Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:171"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.401120298Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_c75afc92-cd90-403e-b6d4-baabf826a6f6/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.408826072Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"Looking for unencrypted datasets in oxp_c75afc92-cd90-403e-b6d4-baabf826a6f6","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.529953099Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.537931095Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:478","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.537977326Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:395","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.537983306Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:380","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.537990386Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:457"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.537994446Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.547901654Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.547929964Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"e6f9779f-728b-405d-a726-810b70de4ab6 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.643844617Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","zpool":"ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.65234234Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.65237502Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.65238782Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.652413811Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.65242234Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.652618893Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.652627003Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/0dcbb472-7cd1-4e42-9116-fc1cc27da573): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:30.656429674Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.691341204Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_e6f9779f-728b-405d-a726-810b70de4ab6/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.699693065Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"Looking for unencrypted datasets in oxp_e6f9779f-728b-405d-a726-810b70de4ab6","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.827324893Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.836018957Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:478","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.836069808Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:395","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.836078758Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:380","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.836088908Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:457"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.836094088Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.845626612Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.845664622Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool)"}
{"msg":"Ensuring tfportrear0_0/ll exists in zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.922125433Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2244"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.942225802Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","zpool":"ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.951068358Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.951115299Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.951123339Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.951168529Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.951179359Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.95129837Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.951307321Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.989929811Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_2bc49789-2398-4c5f-be70-ff11292cbf60/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:30.997621514Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"Looking for unencrypted datasets in oxp_2bc49789-2398-4c5f-be70-ff11292cbf60","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.120024915Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.1277667Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:478","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.12782459Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:395","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.12783054Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:380","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.12783719Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:457"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.12784301Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.137376404Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.137401824Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.229198412Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","zpool":"ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.236823555Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.236857016Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.236864696Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.236886816Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.236895256Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.237070948Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.237097498Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.274104821Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_5a2ec449-297e-4e77-989a-dd421ce4d4aa/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.281910025Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/0dcbb472-7cd1-4e42-9116-fc1cc27da573): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-10T23:53:31.374022137Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:171"}
{"msg":"Looking for unencrypted datasets in oxp_5a2ec449-297e-4e77-989a-dd421ce4d4aa","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.405179136Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.413103922Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:478","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.413134723Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:395","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.413139743Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:380","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.413144303Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:457"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.413149293Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.422429674Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.422453844Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.510574772Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","zpool":"ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.517612829Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.517641389Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.517647899Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.51771123Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.51771939Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.517832981Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.517838191Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.551564828Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_4cab71ff-67df-44db-adca-8ef81d5bc209/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.558470303Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"Looking for unencrypted datasets in oxp_4cab71ff-67df-44db-adca-8ef81d5bc209","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.684471523Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.692201497Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:478","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.692239307Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:395","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"}
{"msg":"Writing ledger to /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.692274798Z","hostname":"buskin","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.69242676Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: cac444c1-8feb-42ab-bee7-aa35319fb578 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: c0d1014c-d647-4991-8139-bcd86ea27898 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.69247042Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.69247792Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"}
{"msg":"Writing ledger to /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.692588151Z","hostname":"buskin","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.692960585Z","hostname":"buskin","pid":1555,"uri":"/omicron-physical-disks","method":"PUT","req_id":"6064d9fe-718e-4619-86f4-9a1a2f55d8aa","remote_addr":"[fd00:1122:3344:101::1]:56903","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":12295213,"response_code":"200"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 89504e37-0b08-442d-ace9-40ca97c7ca33, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.723135403Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.723257945Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:48778"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.723593708Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.723600079Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.77433404Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.774373381Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.823808958Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.823851919Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.875078426Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"No ledger in /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.875125596Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.875135936Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:188"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.875180807Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.919541519Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.91960534Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.91961386Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Configuring new Omicron zone: oxz_internal_dns_89504e37-0b08-442d-ace9-40ca97c7ca33","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.972333063Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_internal_dns_5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.975905591Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_internal_dns_53f5ee67-e6ff-49bd-88ae-d78f8180817c","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.979933465Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_internal_dns_89504e37-0b08-442d-ace9-40ca97c7ca33","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.990608731Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_internal_dns_5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:31.997080801Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_internal_dns_53f5ee67-e6ff-49bd-88ae-d78f8180817c","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:32.001466529Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/0dcbb472-7cd1-4e42-9116-fc1cc27da573): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-10T23:53:32.376143083Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:171"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/0dcbb472-7cd1-4e42-9116-fc1cc27da573): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:32.653847813Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"Profile for oxz_internal_dns_89504e37-0b08-442d-ace9-40ca97c7ca33:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService1\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:1::2\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:1::1\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/internal_dns\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value=\"[fd00:1122:3344:1::1]:5353\"/>\n        <propval type=\"astring\" name=\"dns_address\" value=\"[fd00:1122:3344:1::1]:53\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:33.499535848Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:33.499598339Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:1::, width: 64 }"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:33.499780911Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_89504e37-0b08-442d-ace9-40ca97c7ca33","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_internal_dns_5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService2\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:2::2\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:2::1\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/internal_dns\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value=\"[fd00:1122:3344:2::1]:5353\"/>\n        <propval type=\"astring\" name=\"dns_address\" value=\"[fd00:1122:3344:2::1]:53\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:33.985736475Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_internal_dns_53f5ee67-e6ff-49bd-88ae-d78f8180817c:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService3\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:3::2\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:3::1\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/internal_dns\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value=\"[fd00:1122:3344:3::1]:5353\"/>\n        <propval type=\"astring\" name=\"dns_address\" value=\"[fd00:1122:3344:3::1]:53\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:34.473902853Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:34.473982364Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:2::, width: 64 }"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:34.473996734Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:3::, width: 64 }"}
{"msg":"wait for service Some(\"oxz_internal_dns_89504e37-0b08-442d-ace9-40ca97c7ca33\") failed: Property not found. retry in 70.852148ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:34.482414306Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_89504e37-0b08-442d-ace9-40ca97c7ca33","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:34.482452386Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:34.486595482Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_53f5ee67-e6ff-49bd-88ae-d78f8180817c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_internal_dns_89504e37-0b08-442d-ace9-40ca97c7ca33\") failed: Property not found. retry in 114.995796ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:34.562190573Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_89504e37-0b08-442d-ace9-40ca97c7ca33","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d\") failed: Property not found. retry in 71.633146ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:34.609036303Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_53f5ee67-e6ff-49bd-88ae-d78f8180817c\") failed: Property not found. retry in 66.895233ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:34.615109479Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_53f5ee67-e6ff-49bd-88ae-d78f8180817c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_89504e37-0b08-442d-ace9-40ca97c7ca33\") failed: Property not found. retry in 106.206295ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:34.748657721Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_89504e37-0b08-442d-ace9-40ca97c7ca33","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d\") failed: Property not found. retry in 147.508699ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:34.796209948Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_53f5ee67-e6ff-49bd-88ae-d78f8180817c\") failed: Property not found. retry in 92.709759ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:34.908212146Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_53f5ee67-e6ff-49bd-88ae-d78f8180817c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_89504e37-0b08-442d-ace9-40ca97c7ca33\") failed: Property not found. retry in 539.965774ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:34.938194952Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_89504e37-0b08-442d-ace9-40ca97c7ca33","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d\") failed: Property not found. retry in 274.339894ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:34.965415488Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_53f5ee67-e6ff-49bd-88ae-d78f8180817c\") failed: Property not found. retry in 217.692638ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:35.02901092Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_53f5ee67-e6ff-49bd-88ae-d78f8180817c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d\") failed: Property not found. retry in 244.664523ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:35.251236716Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_53f5ee67-e6ff-49bd-88ae-d78f8180817c\") failed: Property not found. retry in 345.456905ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:35.258901969Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_53f5ee67-e6ff-49bd-88ae-d78f8180817c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_89504e37-0b08-442d-ace9-40ca97c7ca33\") failed: Property not found. retry in 1.02601135s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:35.488101512Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_89504e37-0b08-442d-ace9-40ca97c7ca33","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d\") failed: Property not found. retry in 516.640026ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:35.505669313Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_53f5ee67-e6ff-49bd-88ae-d78f8180817c\") failed: Property not found. retry in 799.730581ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:35.613540866Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_53f5ee67-e6ff-49bd-88ae-d78f8180817c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d\") failed: Property not found. retry in 1.226586077s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:36.031912715Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_53f5ee67-e6ff-49bd-88ae-d78f8180817c\") failed: Property not found. retry in 622.836651ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:36.42321917Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_53f5ee67-e6ff-49bd-88ae-d78f8180817c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_89504e37-0b08-442d-ace9-40ca97c7ca33\") failed: Property not found. retry in 1.136344171s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:36.522976655Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_89504e37-0b08-442d-ace9-40ca97c7ca33","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Ensuring bootstrap address fdb0:18c0:4d0d:9fb2::2 exists in switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:37.003118776Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2261"}
{"msg":"Adding bootstrap address","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:37.003146316Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:626"}
{"msg":"wait for service Some(\"oxz_internal_dns_53f5ee67-e6ff-49bd-88ae-d78f8180817c\") failed: Property not found. retry in 1.199719337s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:37.056576597Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_53f5ee67-e6ff-49bd-88ae-d78f8180817c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d\") failed: Property not found. retry in 903.271274ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:37.266419379Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Forwarding bootstrap traffic via oxBootstrap0 to fe80::8:20ff:fe4b:fd54","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:37.552775373Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2268"}
{"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:37.558330013Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2329"}
{"msg":"wait for service Some(\"oxz_internal_dns_89504e37-0b08-442d-ace9-40ca97c7ca33\") failed: Property not found. retry in 1.079840691s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:37.667000705Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_89504e37-0b08-442d-ace9-40ca97c7ca33","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:37.677269407Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2514"}
{"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:37.917716861Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2678"}
{"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:38.108264293Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2393"}
{"msg":"wait for service Some(\"oxz_internal_dns_5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d\") failed: Property not found. retry in 712.78477ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:38.179632109Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_53f5ee67-e6ff-49bd-88ae-d78f8180817c\") failed: Property not found. retry in 1.130197386s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:38.264059187Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_53f5ee67-e6ff-49bd-88ae-d78f8180817c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:38.408892602Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2428"}
{"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:38.620317672Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2728"}
{"msg":"wait for service Some(\"oxz_internal_dns_89504e37-0b08-442d-ace9-40ca97c7ca33\") failed: Property not found. retry in 841.909161ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:38.755920766Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_89504e37-0b08-442d-ace9-40ca97c7ca33","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:38.800042826Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2755"}
{"msg":"wait for service Some(\"oxz_internal_dns_5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d\") failed: Property not found. retry in 638.893217ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:38.903818584Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:39.036457536Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2622"}
{"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:39.228796007Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2422"}
{"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:39.240912749Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3824","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"}
{"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:39.240960549Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3840"}
{"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:39.24097619Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"}
{"msg":"wait for service Some(\"oxz_internal_dns_53f5ee67-e6ff-49bd-88ae-d78f8180817c\") failed: Property not found. retry in 1.171465055s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:39.402977051Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_53f5ee67-e6ff-49bd-88ae-d78f8180817c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:39.794272736Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3848"}
{"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:39.800952579Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3904"}
{"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:39.945796154Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3966"}
{"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.044981422Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3947","rack_subnet":"fd00:1122:3344::"}
{"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.067051162Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3994"}
{"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.164194299Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4021"}
{"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.276976835Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:369"}
{"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.308370196Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:417"}
{"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.339344573Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:656"}
{"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.339420284Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:694"}
{"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.339832629Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:697","uptime":"3"}
{"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.339857109Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:439","config":"PortSettings {\n    links: {\n        \"0\": LinkSettings {\n            addrs: [\n                10.151.1.174,\n            ],\n            params: LinkCreate {\n                autoneg: false,\n                fec: None,\n                kr: false,\n                lane: Some(\n                    LinkId(\n                        0,\n                    ),\n                ),\n                speed: Speed40G,\n            },\n        },\n    },\n}"}
{"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.415984157Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.416078798Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:564","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.416089888Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:585","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.416098568Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:564","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.416106568Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:585","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.416113478Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:564","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.416120368Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:585","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.416127258Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:564","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.416135248Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:585","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.416141778Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:564","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.416148578Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:585","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.416156358Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:564","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.416162879Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:585","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.416180129Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:564","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.416186889Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:585","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.416194559Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:564","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.416202519Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:585","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.416207749Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:564","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.416212769Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:585","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.416217789Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:564","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.416222889Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:585","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.416227919Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:564","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.416232949Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:585","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.593079682Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_internal_dns_89504e37-0b08-442d-ace9-40ca97c7ca33"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.593135443Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_internal_dns_5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.593145253Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_internal_dns_53f5ee67-e6ff-49bd-88ae-d78f8180817c"}
{"msg":"Writing ledger to /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.593229464Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.593591668Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.593866931Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"9f776f1d-c35c-4cbf-b78f-43c58c346120","remote_addr":"[fd00:1122:3344:101::1]:48778","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":8870526,"response_code":"204"}
{"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.595475488Z","hostname":"buskin","pid":1555,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:549"}
{"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.641974634Z","hostname":"buskin","pid":1555,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:549"}
{"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.688360428Z","hostname":"buskin","pid":1555,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:549"}
{"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.732056723Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:585"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.732067264Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:263"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.763483885Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.764176393Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:305"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.764185813Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 89504e37-0b08-442d-ace9-40ca97c7ca33, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5ae75945-01ab-454a-af38-a5dd1d09d270, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a390c63c-a68b-4301-b3c7-797628a3170a, kind: Service { id: 5ae75945-01ab-454a-af38-a5dd1d09d270 }, name: Name(\"ntp-5ae75945-01ab-454a-af38-a5dd1d09d270\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 108])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.792062596Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.792140787Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:44620"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.792814234Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.792819464Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.812476848Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.812519738Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.8319562Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.83202125Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.850102687Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Reading ledger from /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.850156948Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.850299419Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.850421161Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.860989996Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.892612079Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.893287687Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.893306267Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.937383696Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:478","port":"Port { inner: PortInner(PortData { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 174, 108]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 }, vnic: \"vopte0\" }) }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.937448527Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1200","zone_type":"ntp"}
{"msg":"no nat entry found for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.937942102Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"}
{"msg":"creating nat entry for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.937956773Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"}
{"msg":"creation of nat entry successful for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.938253816Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.939216996Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Configuring new Omicron zone: oxz_ntp_5ae75945-01ab-454a-af38-a5dd1d09d270","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.963137266Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_ntp_5ae75945-01ab-454a-af38-a5dd1d09d270","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:40.983243855Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Profile for oxz_ntp_5ae75945-01ab-454a-af38-a5dd1d09d270:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService4\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::1b\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/chrony-setup\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"allow\" value=\"fd00:1122:3344:100::/56\"/>\n        <propval type=\"boolean\" name=\"boundary\" value=\"true\"/>\n        <propval type=\"astring\" name=\"server\" value=\"0.pool.ntp.org\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"1.1.1.1\" />\n            <value_node value=\"9.9.9.9\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/ntp\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value=\"vopte0\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"172.30.3.1\"/>\n        <propval type=\"astring\" name=\"ip\" value=\"172.30.3.5\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:41.614745832Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:41.614898914Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_5ae75945-01ab-454a-af38-a5dd1d09d270","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_ntp_5ae75945-01ab-454a-af38-a5dd1d09d270\") failed: Property not found. retry in 66.25636ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:41.751315547Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_5ae75945-01ab-454a-af38-a5dd1d09d270","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_5ae75945-01ab-454a-af38-a5dd1d09d270\") failed: Property not found. retry in 68.128091ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:41.834915656Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_5ae75945-01ab-454a-af38-a5dd1d09d270","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_5ae75945-01ab-454a-af38-a5dd1d09d270\") failed: Property not found. retry in 215.389512ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:41.91994128Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_5ae75945-01ab-454a-af38-a5dd1d09d270","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_5ae75945-01ab-454a-af38-a5dd1d09d270\") failed: Property not found. retry in 252.555172ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:42.145842886Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_5ae75945-01ab-454a-af38-a5dd1d09d270","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_5ae75945-01ab-454a-af38-a5dd1d09d270\") failed: Property not found. retry in 790.26443ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:42.406220827Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_5ae75945-01ab-454a-af38-a5dd1d09d270","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_5ae75945-01ab-454a-af38-a5dd1d09d270\") failed: Property not found. retry in 1.415625999s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:43.20652889Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_5ae75945-01ab-454a-af38-a5dd1d09d270","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:43.700104777Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:45616"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:43.700677893Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"9c8564d6-9abc-4c3d-bfef-29a8815666c9","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:45616","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":510,"response_code":"200"}
{"msg":"wait for service Some(\"oxz_ntp_5ae75945-01ab-454a-af38-a5dd1d09d270\") failed: Property not found. retry in 521.467513ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:44.631033349Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_5ae75945-01ab-454a-af38-a5dd1d09d270","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_5ae75945-01ab-454a-af38-a5dd1d09d270\") failed: Property not found. retry in 1.376437185s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:45.162586899Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_5ae75945-01ab-454a-af38-a5dd1d09d270","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_5ae75945-01ab-454a-af38-a5dd1d09d270\") failed: Property not found. retry in 1.364675851s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:46.550052596Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_5ae75945-01ab-454a-af38-a5dd1d09d270","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:47.934297868Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_ntp_5ae75945-01ab-454a-af38-a5dd1d09d270"}
{"msg":"Writing ledger to /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:47.934378749Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:47.934755073Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:47.935078756Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"6e3e07c5-6599-4aa1-9e51-46875c918d66","remote_addr":"[fd00:1122:3344:101::1]:44620","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":7142876,"response_code":"204"}
{"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:47.936887516Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:626"}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:47.965049272Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:47.965190384Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:34967"}
{"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_5ae75945-01ab-454a-af38-a5dd1d09d270': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1  stdout: 506 Cannot talk to daemon\n  stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:47.972757426Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3554"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:47.972900897Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"5687827e-ee45-423d-8d46-21743776cefe","remote_addr":"[fd00:1122:3344:101::1]:34967","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":7618,"response_code":"500"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:47.974280563Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:48.150758422Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:48.150900083Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:40894"}
{"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_5ae75945-01ab-454a-af38-a5dd1d09d270': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1  stdout: 506 Cannot talk to daemon\n  stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:52.159375549Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3554"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:52.159574751Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"d86ad222-19b0-4315-8373-f23896095836","remote_addr":"[fd00:1122:3344:101::1]:40894","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4008511,"response_code":"500"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:52.16315495Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:52.267512775Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:52.267659057Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:46368"}
{"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_5ae75945-01ab-454a-af38-a5dd1d09d270': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1  stdout: 506 Cannot talk to daemon\n  stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:52.277044309Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3554"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:52.27719593Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"630b9eca-3d73-4a6e-9a00-7fe56c4e3ecb","remote_addr":"[fd00:1122:3344:101::1]:46368","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":9399,"response_code":"500"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:52.280320484Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:52.517127589Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:52.517274351Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:42580"}
{"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_5ae75945-01ab-454a-af38-a5dd1d09d270': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1  stdout: 506 Cannot talk to daemon\n  stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:52.525667942Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3554"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:52.525793564Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"e53a193b-4672-43e5-b17a-a85a794ef2fc","remote_addr":"[fd00:1122:3344:101::1]:42580","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8440,"response_code":"500"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:52.528719206Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:52.684042784Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:52.684167176Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:42105"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:56.693776616Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"68bcb931-34c0-4310-9a90-7b4d6992371f","remote_addr":"[fd00:1122:3344:101::1]:42105","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":4009469,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:56.696758918Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:56.696771338Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.032188035Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.032317187Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:63673"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.040842519Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"60dbdc07-ecd5-4fb8-98f6-02b8926f86b2","remote_addr":"[fd00:1122:3344:101::1]:63673","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":8361,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.043861792Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:57.043872252Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.401949386Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.402081107Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:60713"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.410974944Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"f080d19d-5246-4720-a835-a14728792e37","remote_addr":"[fd00:1122:3344:101::1]:60713","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":8781,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.414105868Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:57.414116188Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.622588085Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.622717467Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:40241"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.631380601Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"343a6991-0a27-4ff1-a3fa-92bb6f80784c","remote_addr":"[fd00:1122:3344:101::1]:40241","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":8529,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.633941219Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:57.633951779Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.191032938Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.19120913Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:42646"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.211787743Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"0a10dc9a-ac5b-4305-8b91-36b586ed90b9","remote_addr":"[fd00:1122:3344:101::1]:42646","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":20400,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 2180654075, ip_addr: 129.250.35.251, stratum: 3, ref_time: 1720655636.787068, correction: -0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.213414311Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"}
{"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.213426481Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1224"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 89504e37-0b08-442d-ace9-40ca97c7ca33, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5ae75945-01ab-454a-af38-a5dd1d09d270, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a390c63c-a68b-4301-b3c7-797628a3170a, kind: Service { id: 5ae75945-01ab-454a-af38-a5dd1d09d270 }, name: Name(\"ntp-5ae75945-01ab-454a-af38-a5dd1d09d270\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 108])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.241505586Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.241631408Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:51200"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.242097253Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.242105003Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.261505444Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.261548904Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.28137283Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.281417831Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.297860169Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.29790669Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.350258989Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.350298Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.407857115Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.407907296Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.460084863Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.460125104Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.510212368Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.510269049Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.562400306Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Reading ledger from /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.562447316Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.562704509Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.56280878Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.643542458Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.643591759Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.643598029Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.643603399Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.643608819Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_08c14dbf-bb82-4151-bf37-eb74f12a90ee","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.7365299Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_49e6eea4-7bb4-41e1-bdad-3af89cf5dce4","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.740834706Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_3d5a475d-f26b-4201-bcf1-199b682d3f89","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.744769909Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_31df8de9-1afd-4bb7-abe2-60daf0674d5e","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.749228038Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_87f22eac-e992-48b8-979d-a5ce8a11b65f","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.753844658Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_08c14dbf-bb82-4151-bf37-eb74f12a90ee","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.75861212Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_49e6eea4-7bb4-41e1-bdad-3af89cf5dce4","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.763361751Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_3d5a475d-f26b-4201-bcf1-199b682d3f89","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.768135303Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_31df8de9-1afd-4bb7-abe2-60daf0674d5e","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.772840084Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_87f22eac-e992-48b8-979d-a5ce8a11b65f","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:53:57.777436104Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:53:57.959757957Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:423","component":"producer-server","component":"MetricsManager","sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"oximeter/producer/src/lib.rs:423","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"170.951331ms"}
{"msg":"Profile for oxz_cockroachdb_87f22eac-e992-48b8-979d-a5ce8a11b65f:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService9\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::7\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroachdb\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"[fd00:1122:3344:101::7]:32221\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroach-admin\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"zone_id\" value=\"87f22eac-e992-48b8-979d-a5ce8a11b65f\"/>\n        <propval type=\"astring\" name=\"cockroach_address\" value=\"[fd00:1122:3344:101::7]:32221\"/>\n        <propval type=\"astring\" name=\"http_address\" value=\"[fd00:1122:3344:101::7]:32222\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:3::1\" />\n            <value_node value=\"fd00:1122:3344:2::1\" />\n            <value_node value=\"fd00:1122:3344:1::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:00.782799152Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:00.782974894Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_87f22eac-e992-48b8-979d-a5ce8a11b65f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_cockroachdb_31df8de9-1afd-4bb7-abe2-60daf0674d5e:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService8\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::4\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroachdb\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"[fd00:1122:3344:101::4]:32221\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroach-admin\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"zone_id\" value=\"31df8de9-1afd-4bb7-abe2-60daf0674d5e\"/>\n        <propval type=\"astring\" name=\"cockroach_address\" value=\"[fd00:1122:3344:101::4]:32221\"/>\n        <propval type=\"astring\" name=\"http_address\" value=\"[fd00:1122:3344:101::4]:32222\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:3::1\" />\n            <value_node value=\"fd00:1122:3344:2::1\" />\n            <value_node value=\"fd00:1122:3344:1::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:00.789129181Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:00.789257153Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_31df8de9-1afd-4bb7-abe2-60daf0674d5e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_cockroachdb_3d5a475d-f26b-4201-bcf1-199b682d3f89:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService7\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::5\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroachdb\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"[fd00:1122:3344:101::5]:32221\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroach-admin\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"zone_id\" value=\"3d5a475d-f26b-4201-bcf1-199b682d3f89\"/>\n        <propval type=\"astring\" name=\"cockroach_address\" value=\"[fd00:1122:3344:101::5]:32221\"/>\n        <propval type=\"astring\" name=\"http_address\" value=\"[fd00:1122:3344:101::5]:32222\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:3::1\" />\n            <value_node value=\"fd00:1122:3344:2::1\" />\n            <value_node value=\"fd00:1122:3344:1::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:00.844699095Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:00.844796957Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3d5a475d-f26b-4201-bcf1-199b682d3f89","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_cockroachdb_49e6eea4-7bb4-41e1-bdad-3af89cf5dce4:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService6\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::6\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroachdb\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"[fd00:1122:3344:101::6]:32221\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroach-admin\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"zone_id\" value=\"49e6eea4-7bb4-41e1-bdad-3af89cf5dce4\"/>\n        <propval type=\"astring\" name=\"cockroach_address\" value=\"[fd00:1122:3344:101::6]:32221\"/>\n        <propval type=\"astring\" name=\"http_address\" value=\"[fd00:1122:3344:101::6]:32222\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:3::1\" />\n            <value_node value=\"fd00:1122:3344:2::1\" />\n            <value_node value=\"fd00:1122:3344:1::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:00.897473229Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:00.897625741Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_49e6eea4-7bb4-41e1-bdad-3af89cf5dce4","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_cockroachdb_08c14dbf-bb82-4151-bf37-eb74f12a90ee:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService5\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::3\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroachdb\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"[fd00:1122:3344:101::3]:32221\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroach-admin\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"zone_id\" value=\"08c14dbf-bb82-4151-bf37-eb74f12a90ee\"/>\n        <propval type=\"astring\" name=\"cockroach_address\" value=\"[fd00:1122:3344:101::3]:32221\"/>\n        <propval type=\"astring\" name=\"http_address\" value=\"[fd00:1122:3344:101::3]:32222\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:3::1\" />\n            <value_node value=\"fd00:1122:3344:2::1\" />\n            <value_node value=\"fd00:1122:3344:1::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:00.90676311Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:00.906863941Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_08c14dbf-bb82-4151-bf37-eb74f12a90ee","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_cockroachdb_87f22eac-e992-48b8-979d-a5ce8a11b65f\") failed: Property not found. retry in 61.023553ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:00.935072518Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_87f22eac-e992-48b8-979d-a5ce8a11b65f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_31df8de9-1afd-4bb7-abe2-60daf0674d5e\") failed: Property not found. retry in 25.917708ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:00.94629559Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_31df8de9-1afd-4bb7-abe2-60daf0674d5e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_31df8de9-1afd-4bb7-abe2-60daf0674d5e\") failed: Property not found. retry in 115.370938ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:00.98123542Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_31df8de9-1afd-4bb7-abe2-60daf0674d5e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_3d5a475d-f26b-4201-bcf1-199b682d3f89\") failed: Property not found. retry in 74.732971ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:00.989346048Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3d5a475d-f26b-4201-bcf1-199b682d3f89","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_87f22eac-e992-48b8-979d-a5ce8a11b65f\") failed: Property not found. retry in 141.85472ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:01.016702056Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_87f22eac-e992-48b8-979d-a5ce8a11b65f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_3d5a475d-f26b-4201-bcf1-199b682d3f89\") failed: Property not found. retry in 148.15656ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:01.218187157Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3d5a475d-f26b-4201-bcf1-199b682d3f89","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_31df8de9-1afd-4bb7-abe2-60daf0674d5e\") failed: Property not found. retry in 217.522027ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:01.233796816Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_31df8de9-1afd-4bb7-abe2-60daf0674d5e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_87f22eac-e992-48b8-979d-a5ce8a11b65f\") failed: Property not found. retry in 123.423055ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:01.279085368Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_87f22eac-e992-48b8-979d-a5ce8a11b65f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_3d5a475d-f26b-4201-bcf1-199b682d3f89\") failed: Property not found. retry in 122.212396ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:01.449664863Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3d5a475d-f26b-4201-bcf1-199b682d3f89","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_08c14dbf-bb82-4151-bf37-eb74f12a90ee\") failed: Property not found. retry in 60.049926ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:01.532090779Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_08c14dbf-bb82-4151-bf37-eb74f12a90ee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_49e6eea4-7bb4-41e1-bdad-3af89cf5dce4\") failed: Property not found. retry in 60.466118ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:01.633449201Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_49e6eea4-7bb4-41e1-bdad-3af89cf5dce4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_3d5a475d-f26b-4201-bcf1-199b682d3f89\") failed: Property not found. retry in 456.137868ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:01.66641667Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3d5a475d-f26b-4201-bcf1-199b682d3f89","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_31df8de9-1afd-4bb7-abe2-60daf0674d5e\") failed: Property not found. retry in 222.825128ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:01.696116963Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_31df8de9-1afd-4bb7-abe2-60daf0674d5e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_87f22eac-e992-48b8-979d-a5ce8a11b65f\") failed: Property not found. retry in 546.398913ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:01.737420502Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_87f22eac-e992-48b8-979d-a5ce8a11b65f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_08c14dbf-bb82-4151-bf37-eb74f12a90ee\") failed: Property not found. retry in 134.288477ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:01.768805453Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_08c14dbf-bb82-4151-bf37-eb74f12a90ee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_49e6eea4-7bb4-41e1-bdad-3af89cf5dce4\") failed: Property not found. retry in 120.878874ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:01.798136992Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_49e6eea4-7bb4-41e1-bdad-3af89cf5dce4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_08c14dbf-bb82-4151-bf37-eb74f12a90ee\") failed: Property not found. retry in 111.36853ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:02.073537207Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_08c14dbf-bb82-4151-bf37-eb74f12a90ee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_49e6eea4-7bb4-41e1-bdad-3af89cf5dce4\") failed: Property not found. retry in 201.937613ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:02.14187353Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_49e6eea4-7bb4-41e1-bdad-3af89cf5dce4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_3d5a475d-f26b-4201-bcf1-199b682d3f89\") failed: Property not found. retry in 1.114148805s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:02.182432271Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3d5a475d-f26b-4201-bcf1-199b682d3f89","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_31df8de9-1afd-4bb7-abe2-60daf0674d5e\") failed: Property not found. retry in 991.436107ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:02.215718583Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_31df8de9-1afd-4bb7-abe2-60daf0674d5e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_08c14dbf-bb82-4151-bf37-eb74f12a90ee\") failed: Property not found. retry in 290.103422ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:02.240675994Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_08c14dbf-bb82-4151-bf37-eb74f12a90ee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_87f22eac-e992-48b8-979d-a5ce8a11b65f\") failed: Property not found. retry in 1.10250792s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:02.32857275Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_87f22eac-e992-48b8-979d-a5ce8a11b65f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_49e6eea4-7bb4-41e1-bdad-3af89cf5dce4\") failed: Property not found. retry in 488.934847ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:02.394521327Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_49e6eea4-7bb4-41e1-bdad-3af89cf5dce4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_08c14dbf-bb82-4151-bf37-eb74f12a90ee\") failed: Property not found. retry in 867.010152ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:02.609241822Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_08c14dbf-bb82-4151-bf37-eb74f12a90ee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_49e6eea4-7bb4-41e1-bdad-3af89cf5dce4\") failed: Property not found. retry in 971.663363ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:02.92725466Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_49e6eea4-7bb4-41e1-bdad-3af89cf5dce4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_31df8de9-1afd-4bb7-abe2-60daf0674d5e\") failed: Property not found. retry in 1.119714287s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:03.31624931Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_31df8de9-1afd-4bb7-abe2-60daf0674d5e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_3d5a475d-f26b-4201-bcf1-199b682d3f89\") failed: Property not found. retry in 1.10887491s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:03.366213463Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3d5a475d-f26b-4201-bcf1-199b682d3f89","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_87f22eac-e992-48b8-979d-a5ce8a11b65f\") failed: Property not found. retry in 1.318429799s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:03.445347334Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_87f22eac-e992-48b8-979d-a5ce8a11b65f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_08c14dbf-bb82-4151-bf37-eb74f12a90ee\") failed: Property not found. retry in 930.608995ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:03.487611773Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_08c14dbf-bb82-4151-bf37-eb74f12a90ee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_49e6eea4-7bb4-41e1-bdad-3af89cf5dce4\") failed: Property not found. retry in 1.072214627s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:03.911045447Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_49e6eea4-7bb4-41e1-bdad-3af89cf5dce4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_08c14dbf-bb82-4151-bf37-eb74f12a90ee\") failed: Property not found. retry in 830.369825ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:04.428120389Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_08c14dbf-bb82-4151-bf37-eb74f12a90ee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_31df8de9-1afd-4bb7-abe2-60daf0674d5e\") failed: Property not found. retry in 1.488061928s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:04.444535138Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_31df8de9-1afd-4bb7-abe2-60daf0674d5e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_3d5a475d-f26b-4201-bcf1-199b682d3f89\") failed: Property not found. retry in 631.084993ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:04.485701295Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3d5a475d-f26b-4201-bcf1-199b682d3f89","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_87f22eac-e992-48b8-979d-a5ce8a11b65f\") failed: Property not found. retry in 657.794315ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:04.77304627Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_87f22eac-e992-48b8-979d-a5ce8a11b65f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_49e6eea4-7bb4-41e1-bdad-3af89cf5dce4\") failed: Property not found. retry in 1.420413926s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:04.992157332Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_49e6eea4-7bb4-41e1-bdad-3af89cf5dce4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_3d5a475d-f26b-4201-bcf1-199b682d3f89\") failed: Property not found. retry in 983.046644ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:05.126422842Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3d5a475d-f26b-4201-bcf1-199b682d3f89","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_08c14dbf-bb82-4151-bf37-eb74f12a90ee\") failed: Property not found. retry in 1.116123449s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:05.267889361Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_08c14dbf-bb82-4151-bf37-eb74f12a90ee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_87f22eac-e992-48b8-979d-a5ce8a11b65f\") failed: Property not found. retry in 602.848623ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:05.440172344Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_87f22eac-e992-48b8-979d-a5ce8a11b65f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_31df8de9-1afd-4bb7-abe2-60daf0674d5e\") failed: Property not found. retry in 501.851469ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:05.941099141Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_31df8de9-1afd-4bb7-abe2-60daf0674d5e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_87f22eac-e992-48b8-979d-a5ce8a11b65f\") failed: Property not found. retry in 655.106131ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:06.052817176Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_87f22eac-e992-48b8-979d-a5ce8a11b65f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_3d5a475d-f26b-4201-bcf1-199b682d3f89\") failed: Property not found. retry in 911.901955ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:06.118886554Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3d5a475d-f26b-4201-bcf1-199b682d3f89","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_08c14dbf-bb82-4151-bf37-eb74f12a90ee\") failed: Property not found. retry in 859.07313ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:06.396410472Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_08c14dbf-bb82-4151-bf37-eb74f12a90ee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_49e6eea4-7bb4-41e1-bdad-3af89cf5dce4\") failed: Property not found. retry in 505.92669ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:06.424979542Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_49e6eea4-7bb4-41e1-bdad-3af89cf5dce4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_31df8de9-1afd-4bb7-abe2-60daf0674d5e\") failed: Property not found. retry in 1.160537899s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:06.45422378Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_31df8de9-1afd-4bb7-abe2-60daf0674d5e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_87f22eac-e992-48b8-979d-a5ce8a11b65f\") failed: Property not found. retry in 776.440131ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:06.718737397Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_87f22eac-e992-48b8-979d-a5ce8a11b65f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_49e6eea4-7bb4-41e1-bdad-3af89cf5dce4\") failed: Property not found. retry in 1.497555564s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:06.94160334Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_49e6eea4-7bb4-41e1-bdad-3af89cf5dce4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_3d5a475d-f26b-4201-bcf1-199b682d3f89\") failed: Property not found. retry in 909.137711ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:07.039883839Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3d5a475d-f26b-4201-bcf1-199b682d3f89","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_08c14dbf-bb82-4151-bf37-eb74f12a90ee\") failed: Property not found. retry in 1.433601428s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:07.266637164Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_08c14dbf-bb82-4151-bf37-eb74f12a90ee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_87f22eac-e992-48b8-979d-a5ce8a11b65f\") failed: Property not found. retry in 1.158097762s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:07.503721862Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_87f22eac-e992-48b8-979d-a5ce8a11b65f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_31df8de9-1afd-4bb7-abe2-60daf0674d5e\") failed: Property not found. retry in 994.882893ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:07.623694366Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_31df8de9-1afd-4bb7-abe2-60daf0674d5e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_3d5a475d-f26b-4201-bcf1-199b682d3f89\") failed: Property not found. retry in 1.20413835s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:07.957329074Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3d5a475d-f26b-4201-bcf1-199b682d3f89","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_49e6eea4-7bb4-41e1-bdad-3af89cf5dce4\") failed: Property not found. retry in 858.606175ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:08.449304304Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_49e6eea4-7bb4-41e1-bdad-3af89cf5dce4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_31df8de9-1afd-4bb7-abe2-60daf0674d5e\") failed: Property not found. retry in 1.429006728s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:08.62919529Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_31df8de9-1afd-4bb7-abe2-60daf0674d5e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_87f22eac-e992-48b8-979d-a5ce8a11b65f\") failed: Property not found. retry in 784.06964ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:08.673233169Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_87f22eac-e992-48b8-979d-a5ce8a11b65f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_08c14dbf-bb82-4151-bf37-eb74f12a90ee\") failed: Property not found. retry in 1.28844572s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:08.709500963Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_08c14dbf-bb82-4151-bf37-eb74f12a90ee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_3d5a475d-f26b-4201-bcf1-199b682d3f89\") failed: Property not found. retry in 1.383929018s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:09.1717414Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3d5a475d-f26b-4201-bcf1-199b682d3f89","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_49e6eea4-7bb4-41e1-bdad-3af89cf5dce4\") failed: Property not found. retry in 874.670171ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:09.31801394Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_49e6eea4-7bb4-41e1-bdad-3af89cf5dce4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_87f22eac-e992-48b8-979d-a5ce8a11b65f\") failed: Property not found. retry in 1.024241799s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:09.466822938Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_87f22eac-e992-48b8-979d-a5ce8a11b65f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:10.576351453Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_cockroachdb_08c14dbf-bb82-4151-bf37-eb74f12a90ee"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:10.576444124Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_cockroachdb_49e6eea4-7bb4-41e1-bdad-3af89cf5dce4"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:10.576455944Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_cockroachdb_3d5a475d-f26b-4201-bcf1-199b682d3f89"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:10.576462834Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_cockroachdb_31df8de9-1afd-4bb7-abe2-60daf0674d5e"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:10.576471354Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_cockroachdb_87f22eac-e992-48b8-979d-a5ce8a11b65f"}
{"msg":"Writing ledger to /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:10.576476734Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:10.576833708Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:10.577154291Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"7de0ad93-cab9-438f-b504-9b1c179b1a81","remote_addr":"[fd00:1122:3344:101::1]:51200","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":13304162,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:10.610002919Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:50905"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:10.637617859Z","hostname":"buskin","pid":1555,"uri":"/cockroachdb","method":"POST","req_id":"665fb818-47f3-4cc8-a1e2-62fbdc604ec8","remote_addr":"[fd00:1122:3344:101::1]:50905","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService5/omicron6 in oxz_cockroachdb_08c14dbf-bb82-4151-bf37-eb74f12a90ee: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_08c14dbf-bb82-4151-bf37-eb74f12a90ee /usr/sbin/ipadm show-addr -p -o ADDR oxControlService5/omicron6] executed and failed with status: exit status: 1  stdout: \n  stderr: ipadm: Address object not found","latency_us":27486,"response_code":"500"}
{"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:10.637830471Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:966","retry_after":"119.61881ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"665fb818-47f3-4cc8-a1e2-62fbdc604ec8\", \"content-length\": \"124\", \"date\": \"Wed, 10 Jul 2024 23:54:10 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"665fb818-47f3-4cc8-a1e2-62fbdc604ec8\" }"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:10.785375476Z","hostname":"buskin","pid":1555,"uri":"/cockroachdb","method":"POST","req_id":"436c14dd-97ed-455e-a87e-e11fee4f8202","remote_addr":"[fd00:1122:3344:101::1]:50905","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService5/omicron6 in oxz_cockroachdb_08c14dbf-bb82-4151-bf37-eb74f12a90ee: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_08c14dbf-bb82-4151-bf37-eb74f12a90ee /usr/sbin/ipadm show-addr -p -o ADDR oxControlService5/omicron6] executed and failed with status: exit status: 1  stdout: \n  stderr: ipadm: Address object not found","latency_us":27287,"response_code":"500"}
{"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:10.785532467Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:966","retry_after":"156.713491ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"436c14dd-97ed-455e-a87e-e11fee4f8202\", \"content-length\": \"124\", \"date\": \"Wed, 10 Jul 2024 23:54:10 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"436c14dd-97ed-455e-a87e-e11fee4f8202\" }"}
{"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:10.984266098Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3388"}
{"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:30.042214004Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3406"}
{"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:42.874090671Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3429"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:42.874602527Z","hostname":"buskin","pid":1555,"uri":"/cockroachdb","method":"POST","req_id":"8035dfa4-2d5a-4c87-bb87-65a04a96f7eb","remote_addr":"[fd00:1122:3344:101::1]:50905","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":31930300,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:42.876179584Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:47014"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:42.881554723Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:423","component":"producer-server","component":"MetricsManager","sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"oximeter/producer/src/lib.rs:423","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"418.954865ms"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 89504e37-0b08-442d-ace9-40ca97c7ca33, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5ae75945-01ab-454a-af38-a5dd1d09d270, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a390c63c-a68b-4301-b3c7-797628a3170a, kind: Service { id: 5ae75945-01ab-454a-af38-a5dd1d09d270 }, name: Name(\"ntp-5ae75945-01ab-454a-af38-a5dd1d09d270\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 108])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac507aba-8f76-47c6-bd22-6dff420215b2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2811064c-b3b2-438e-8e0c-aa9a3bfa436c, kind: Service { id: ac507aba-8f76-47c6-bd22-6dff420215b2 }, name: Name(\"external-dns-ac507aba-8f76-47c6-bd22-6dff420215b2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 139])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b39a7e8e-2dea-45da-9a7a-6977886585dc, kind: Service { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa }, name: Name(\"external-dns-b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 77])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: c108631e-f757-4a42-b510-28307153f426, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6ab9e3b5-f385-4e7e-9bc6-6c4f7a443782, kind: Service { id: c108631e-f757-4a42-b510-28307153f426 }, name: Name(\"nexus-c108631e-f757-4a42-b510-28307153f426\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 191])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6caaab71-9cdd-4faa-b082-987c9dd43604, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: ae27ee1d-0f9e-4fca-b104-9220b85a9848, kind: Service { id: 6caaab71-9cdd-4faa-b082-987c9dd43604 }, name: Name(\"nexus-6caaab71-9cdd-4faa-b082-987c9dd43604\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 147])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 37d6282c-2511-4c34-b419-e68b280bef44, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 97294434-1769-4a97-a97a-20d887e8c50a, kind: Service { id: 37d6282c-2511-4c34-b419-e68b280bef44 }, name: Name(\"nexus-37d6282c-2511-4c34-b419-e68b280bef44\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 88])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: abb1e589-7222-4a55-90ee-b0be940b410b, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: f864e092-4084-4307-8b83-2fd99a97943d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: fce1706b-b90a-4a5d-86b7-20320feb3342, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 32f00476-bb7b-4627-82ad-33c437d3a384, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: e736985c-90fa-417e-89f0-76ea8e60041f, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5baef1a-4fec-4af8-b323-da15f9e7b116, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 21592584-d863-4125-9acf-64a36ff982d8, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:42.915064697Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:42.915501332Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:63238"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:42.916223569Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:42.91623058Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:42.940785676Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:42.940840107Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:42.96036168Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:42.96039969Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:42.979048963Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:42.979093143Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:42.996575893Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:42.996622124Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:43.016194116Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:43.016235647Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:43.034376774Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:43.034423055Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:43.052574552Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:43.052612812Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:43.070429756Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:43.070474347Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:43.172011061Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:43.172054581Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:43.276030232Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:43.276090782Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:43.397976828Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:43.398038299Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:43.520197417Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:43.520242367Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:43.640401774Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:43.640449235Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:43.776307802Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:43.776365022Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:43.909000135Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:43.909065735Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:44.041273353Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:44.041422555Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:44.196944646Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:44.197000976Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:44.328402234Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:44.328460765Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:44.492942964Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:44.493004474Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:44.577570174Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Reading ledger from /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:44.577629134Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:44.578081889Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:44.578399563Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:44.689288819Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:44.689336649Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:44.689343149Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:44.689347759Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:44.689352119Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:44.689355479Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:44.689359639Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:44.689362989Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:44.689367259Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:44.689370729Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:44.68937496Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:44.68938868Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:44.68939321Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:44.68939747Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:44.68940169Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:44.68940718Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:44.68941161Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:44.68941569Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:44.68941994Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:44.689572182Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"93efdc83-e958-47e8-adbb-0683e1cf2b79","remote_addr":"[fd00:1122:3344:101::1]:63238","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\", TimeNotSynchronized), (\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\", TimeNotSynchronized), (\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\", TimeNotSynchronized), (\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\", TimeNotSynchronized), (\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\", TimeNotSynchronized), (\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\", TimeNotSynchronized), (\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\", TimeNotSynchronized), (\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\", TimeNotSynchronized), (\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\", TimeNotSynchronized), (\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\", TimeNotSynchronized), (\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\", TimeNotSynchronized), (\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\", TimeNotSynchronized), (\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\", TimeNotSynchronized), (\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\", TimeNotSynchronized), (\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\", TimeNotSynchronized), (\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\", TimeNotSynchronized), (\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\", TimeNotSynchronized), (\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\", TimeNotSynchronized), (\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\", TimeNotSynchronized)]","latency_us":1773849,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:44.690022896Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"95.030923ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"93efdc83-e958-47e8-adbb-0683e1cf2b79\", \"content-length\": \"124\", \"date\": \"Wed, 10 Jul 2024 23:54:44 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"93efdc83-e958-47e8-adbb-0683e1cf2b79\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 89504e37-0b08-442d-ace9-40ca97c7ca33, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5ae75945-01ab-454a-af38-a5dd1d09d270, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a390c63c-a68b-4301-b3c7-797628a3170a, kind: Service { id: 5ae75945-01ab-454a-af38-a5dd1d09d270 }, name: Name(\"ntp-5ae75945-01ab-454a-af38-a5dd1d09d270\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 108])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac507aba-8f76-47c6-bd22-6dff420215b2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2811064c-b3b2-438e-8e0c-aa9a3bfa436c, kind: Service { id: ac507aba-8f76-47c6-bd22-6dff420215b2 }, name: Name(\"external-dns-ac507aba-8f76-47c6-bd22-6dff420215b2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 139])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b39a7e8e-2dea-45da-9a7a-6977886585dc, kind: Service { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa }, name: Name(\"external-dns-b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 77])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: c108631e-f757-4a42-b510-28307153f426, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6ab9e3b5-f385-4e7e-9bc6-6c4f7a443782, kind: Service { id: c108631e-f757-4a42-b510-28307153f426 }, name: Name(\"nexus-c108631e-f757-4a42-b510-28307153f426\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 191])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6caaab71-9cdd-4faa-b082-987c9dd43604, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: ae27ee1d-0f9e-4fca-b104-9220b85a9848, kind: Service { id: 6caaab71-9cdd-4faa-b082-987c9dd43604 }, name: Name(\"nexus-6caaab71-9cdd-4faa-b082-987c9dd43604\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 147])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 37d6282c-2511-4c34-b419-e68b280bef44, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 97294434-1769-4a97-a97a-20d887e8c50a, kind: Service { id: 37d6282c-2511-4c34-b419-e68b280bef44 }, name: Name(\"nexus-37d6282c-2511-4c34-b419-e68b280bef44\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 88])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: abb1e589-7222-4a55-90ee-b0be940b410b, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: f864e092-4084-4307-8b83-2fd99a97943d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: fce1706b-b90a-4a5d-86b7-20320feb3342, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 32f00476-bb7b-4627-82ad-33c437d3a384, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: e736985c-90fa-417e-89f0-76ea8e60041f, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5baef1a-4fec-4af8-b323-da15f9e7b116, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 21592584-d863-4125-9acf-64a36ff982d8, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:44.786272653Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:44.787427986Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:44.787445116Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:44.807980099Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:44.8080223Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:44.828632294Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:44.828687144Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:44.847476919Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:44.847521129Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:44.86510507Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:44.865142461Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:44.882102495Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:44.882148526Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:44.901204293Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:44.901248203Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:44.921564304Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:44.921603975Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:44.940387369Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:44.940421089Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:44.959611518Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:44.959650158Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:44.978590624Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:44.978630425Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.00028211Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.000322471Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.020325518Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.020362079Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.040789071Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.040823841Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.059664836Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.059705076Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.077359748Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.077395999Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.100318098Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.100376329Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.123090736Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.123133906Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.14375779Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.143809051Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.163828569Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.163868389Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.182442271Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Reading ledger from /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.182498612Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.182739729Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.182874525Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:45.28170399Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:45.281753001Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:45.281759171Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:45.281762881Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:45.281767111Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:45.281772371Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:45.281776681Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:45.281780081Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:45.281784071Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:45.281789461Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:45.281793781Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:45.281798091Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:45.281803191Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:45.281806491Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:45.281811662Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:45.281825842Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:45.281830092Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:45.281834222Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:45.281838512Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.281917543Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"04a8dd45-5ded-4f08-aaf2-24fca59befb7","remote_addr":"[fd00:1122:3344:101::1]:63238","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\", TimeNotSynchronized), (\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\", TimeNotSynchronized), (\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\", TimeNotSynchronized), (\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\", TimeNotSynchronized), (\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\", TimeNotSynchronized), (\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\", TimeNotSynchronized), (\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\", TimeNotSynchronized), (\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\", TimeNotSynchronized), (\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\", TimeNotSynchronized), (\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\", TimeNotSynchronized), (\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\", TimeNotSynchronized), (\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\", TimeNotSynchronized), (\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\", TimeNotSynchronized), (\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\", TimeNotSynchronized), (\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\", TimeNotSynchronized), (\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\", TimeNotSynchronized), (\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\", TimeNotSynchronized), (\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\", TimeNotSynchronized), (\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\", TimeNotSynchronized)]","latency_us":495438,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:45.282141805Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"75.343458ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"04a8dd45-5ded-4f08-aaf2-24fca59befb7\", \"content-length\": \"124\", \"date\": \"Wed, 10 Jul 2024 23:54:44 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"04a8dd45-5ded-4f08-aaf2-24fca59befb7\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 89504e37-0b08-442d-ace9-40ca97c7ca33, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5ae75945-01ab-454a-af38-a5dd1d09d270, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a390c63c-a68b-4301-b3c7-797628a3170a, kind: Service { id: 5ae75945-01ab-454a-af38-a5dd1d09d270 }, name: Name(\"ntp-5ae75945-01ab-454a-af38-a5dd1d09d270\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 108])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac507aba-8f76-47c6-bd22-6dff420215b2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2811064c-b3b2-438e-8e0c-aa9a3bfa436c, kind: Service { id: ac507aba-8f76-47c6-bd22-6dff420215b2 }, name: Name(\"external-dns-ac507aba-8f76-47c6-bd22-6dff420215b2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 139])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b39a7e8e-2dea-45da-9a7a-6977886585dc, kind: Service { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa }, name: Name(\"external-dns-b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 77])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: c108631e-f757-4a42-b510-28307153f426, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6ab9e3b5-f385-4e7e-9bc6-6c4f7a443782, kind: Service { id: c108631e-f757-4a42-b510-28307153f426 }, name: Name(\"nexus-c108631e-f757-4a42-b510-28307153f426\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 191])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6caaab71-9cdd-4faa-b082-987c9dd43604, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: ae27ee1d-0f9e-4fca-b104-9220b85a9848, kind: Service { id: 6caaab71-9cdd-4faa-b082-987c9dd43604 }, name: Name(\"nexus-6caaab71-9cdd-4faa-b082-987c9dd43604\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 147])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 37d6282c-2511-4c34-b419-e68b280bef44, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 97294434-1769-4a97-a97a-20d887e8c50a, kind: Service { id: 37d6282c-2511-4c34-b419-e68b280bef44 }, name: Name(\"nexus-37d6282c-2511-4c34-b419-e68b280bef44\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 88])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: abb1e589-7222-4a55-90ee-b0be940b410b, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: f864e092-4084-4307-8b83-2fd99a97943d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: fce1706b-b90a-4a5d-86b7-20320feb3342, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 32f00476-bb7b-4627-82ad-33c437d3a384, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: e736985c-90fa-417e-89f0-76ea8e60041f, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5baef1a-4fec-4af8-b323-da15f9e7b116, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 21592584-d863-4125-9acf-64a36ff982d8, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.358605686Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.358980611Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.358992651Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.380955049Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.38100043Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.400033297Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.400083348Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.418854552Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.418907142Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.436986969Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.437027649Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.455272228Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.455311248Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.477055894Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.477100955Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.498620169Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.49866098Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.51899167Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.519041751Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.541837969Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.541879939Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.563319653Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.563357903Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.585603675Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.585650806Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.607785846Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.607825387Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.628995297Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.629038267Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.650620772Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.650666142Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.672802573Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.672840434Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.693975463Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.694012984Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.715225924Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.715270195Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.737274634Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.737312335Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.759559697Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.759599467Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.782380175Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Reading ledger from /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.782421575Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.782636477Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.782771009Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:45.883288932Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:45.883349783Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:45.883358543Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:45.883364693Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:45.883389663Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:45.883396283Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:45.883402163Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:45.883406213Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:45.883412183Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:45.883416433Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:45.883422273Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:45.883427583Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:45.883431913Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:45.883437544Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:45.883443244Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:45.883448734Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:45.883455524Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:45.883460914Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:45.883466424Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:45.883600525Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"77c5407a-f4ae-43fd-afff-af744136efbe","remote_addr":"[fd00:1122:3344:101::1]:63238","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\", TimeNotSynchronized), (\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\", TimeNotSynchronized), (\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\", TimeNotSynchronized), (\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\", TimeNotSynchronized), (\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\", TimeNotSynchronized), (\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\", TimeNotSynchronized), (\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\", TimeNotSynchronized), (\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\", TimeNotSynchronized), (\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\", TimeNotSynchronized), (\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\", TimeNotSynchronized), (\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\", TimeNotSynchronized), (\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\", TimeNotSynchronized), (\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\", TimeNotSynchronized), (\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\", TimeNotSynchronized), (\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\", TimeNotSynchronized), (\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\", TimeNotSynchronized), (\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\", TimeNotSynchronized), (\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\", TimeNotSynchronized), (\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\", TimeNotSynchronized)]","latency_us":524835,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:45.883862268Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"202.053965ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"77c5407a-f4ae-43fd-afff-af744136efbe\", \"content-length\": \"124\", \"date\": \"Wed, 10 Jul 2024 23:54:45 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"77c5407a-f4ae-43fd-afff-af744136efbe\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 89504e37-0b08-442d-ace9-40ca97c7ca33, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5ae75945-01ab-454a-af38-a5dd1d09d270, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a390c63c-a68b-4301-b3c7-797628a3170a, kind: Service { id: 5ae75945-01ab-454a-af38-a5dd1d09d270 }, name: Name(\"ntp-5ae75945-01ab-454a-af38-a5dd1d09d270\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 108])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac507aba-8f76-47c6-bd22-6dff420215b2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2811064c-b3b2-438e-8e0c-aa9a3bfa436c, kind: Service { id: ac507aba-8f76-47c6-bd22-6dff420215b2 }, name: Name(\"external-dns-ac507aba-8f76-47c6-bd22-6dff420215b2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 139])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b39a7e8e-2dea-45da-9a7a-6977886585dc, kind: Service { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa }, name: Name(\"external-dns-b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 77])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: c108631e-f757-4a42-b510-28307153f426, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6ab9e3b5-f385-4e7e-9bc6-6c4f7a443782, kind: Service { id: c108631e-f757-4a42-b510-28307153f426 }, name: Name(\"nexus-c108631e-f757-4a42-b510-28307153f426\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 191])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6caaab71-9cdd-4faa-b082-987c9dd43604, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: ae27ee1d-0f9e-4fca-b104-9220b85a9848, kind: Service { id: 6caaab71-9cdd-4faa-b082-987c9dd43604 }, name: Name(\"nexus-6caaab71-9cdd-4faa-b082-987c9dd43604\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 147])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 37d6282c-2511-4c34-b419-e68b280bef44, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 97294434-1769-4a97-a97a-20d887e8c50a, kind: Service { id: 37d6282c-2511-4c34-b419-e68b280bef44 }, name: Name(\"nexus-37d6282c-2511-4c34-b419-e68b280bef44\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 88])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: abb1e589-7222-4a55-90ee-b0be940b410b, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: f864e092-4084-4307-8b83-2fd99a97943d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: fce1706b-b90a-4a5d-86b7-20320feb3342, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 32f00476-bb7b-4627-82ad-33c437d3a384, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: e736985c-90fa-417e-89f0-76ea8e60041f, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5baef1a-4fec-4af8-b323-da15f9e7b116, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 21592584-d863-4125-9acf-64a36ff982d8, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.087140579Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.087533493Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.087540933Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.110698805Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.110752405Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.130349528Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.130393049Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.15162132Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.15167304Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.172134603Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.172182463Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.193572516Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.193610776Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.215403433Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.215446834Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.235720574Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.235760675Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.256421869Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.25646439Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.277441318Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.277493358Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.298626338Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.298669519Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.318506804Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.318543445Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.337564622Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.337605662Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.359760853Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.359805504Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.382004225Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.382060546Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.401655768Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.401695179Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.421032409Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.42107744Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.441458981Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.441510712Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.46343829Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.463480731Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.484437189Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.484476959Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.506832752Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Reading ledger from /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.506889273Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.507097805Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.507234076Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:46.608291866Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:46.608369386Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:46.608377146Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:46.608385367Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:46.608393947Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:46.608403777Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:46.608413297Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:46.608420977Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:46.608445027Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:46.608450697Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:46.608456577Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:46.608463877Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:46.608469517Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:46.608475017Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:46.608480397Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:46.608487377Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:46.608493008Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:46.608498588Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:46.608504218Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.608597419Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"313d7f7f-7eef-4b24-a6f6-f2d69f689eaf","remote_addr":"[fd00:1122:3344:101::1]:63238","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\", TimeNotSynchronized), (\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\", TimeNotSynchronized), (\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\", TimeNotSynchronized), (\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\", TimeNotSynchronized), (\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\", TimeNotSynchronized), (\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\", TimeNotSynchronized), (\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\", TimeNotSynchronized), (\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\", TimeNotSynchronized), (\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\", TimeNotSynchronized), (\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\", TimeNotSynchronized), (\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\", TimeNotSynchronized), (\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\", TimeNotSynchronized), (\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\", TimeNotSynchronized), (\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\", TimeNotSynchronized), (\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\", TimeNotSynchronized), (\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\", TimeNotSynchronized), (\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\", TimeNotSynchronized), (\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\", TimeNotSynchronized), (\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\", TimeNotSynchronized)]","latency_us":521289,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:46.608863732Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"156.663189ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"313d7f7f-7eef-4b24-a6f6-f2d69f689eaf\", \"content-length\": \"124\", \"date\": \"Wed, 10 Jul 2024 23:54:46 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"313d7f7f-7eef-4b24-a6f6-f2d69f689eaf\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 89504e37-0b08-442d-ace9-40ca97c7ca33, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5ae75945-01ab-454a-af38-a5dd1d09d270, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a390c63c-a68b-4301-b3c7-797628a3170a, kind: Service { id: 5ae75945-01ab-454a-af38-a5dd1d09d270 }, name: Name(\"ntp-5ae75945-01ab-454a-af38-a5dd1d09d270\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 108])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac507aba-8f76-47c6-bd22-6dff420215b2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2811064c-b3b2-438e-8e0c-aa9a3bfa436c, kind: Service { id: ac507aba-8f76-47c6-bd22-6dff420215b2 }, name: Name(\"external-dns-ac507aba-8f76-47c6-bd22-6dff420215b2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 139])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b39a7e8e-2dea-45da-9a7a-6977886585dc, kind: Service { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa }, name: Name(\"external-dns-b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 77])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: c108631e-f757-4a42-b510-28307153f426, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6ab9e3b5-f385-4e7e-9bc6-6c4f7a443782, kind: Service { id: c108631e-f757-4a42-b510-28307153f426 }, name: Name(\"nexus-c108631e-f757-4a42-b510-28307153f426\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 191])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6caaab71-9cdd-4faa-b082-987c9dd43604, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: ae27ee1d-0f9e-4fca-b104-9220b85a9848, kind: Service { id: 6caaab71-9cdd-4faa-b082-987c9dd43604 }, name: Name(\"nexus-6caaab71-9cdd-4faa-b082-987c9dd43604\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 147])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 37d6282c-2511-4c34-b419-e68b280bef44, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 97294434-1769-4a97-a97a-20d887e8c50a, kind: Service { id: 37d6282c-2511-4c34-b419-e68b280bef44 }, name: Name(\"nexus-37d6282c-2511-4c34-b419-e68b280bef44\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 88])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: abb1e589-7222-4a55-90ee-b0be940b410b, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: f864e092-4084-4307-8b83-2fd99a97943d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: fce1706b-b90a-4a5d-86b7-20320feb3342, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 32f00476-bb7b-4627-82ad-33c437d3a384, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: e736985c-90fa-417e-89f0-76ea8e60041f, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5baef1a-4fec-4af8-b323-da15f9e7b116, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 21592584-d863-4125-9acf-64a36ff982d8, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.767394285Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.767734219Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.767741679Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.791098803Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.791159284Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.809734276Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.809785397Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.830676114Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.830729994Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.849362287Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.849413727Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.869197503Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.869256363Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.887769174Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.887813925Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.907251816Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.907300467Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.927039812Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.927089522Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.948280423Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.948325223Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.968187029Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.96824618Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.98853688Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:46.988590741Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.008300965Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.008350966Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.028377683Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.028428404Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.047050437Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.047102507Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.069792144Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.069866174Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.090278827Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.090331307Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.108859029Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.108911359Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.127130817Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.127181158Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.147728381Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.147780552Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.169937373Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Reading ledger from /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.169991813Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.170186505Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.170317557Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:47.269852509Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:47.26992197Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:47.26992945Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:47.26993737Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:47.2699417Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:47.26994757Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:47.26995184Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:47.26995724Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:47.26996322Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:47.26996848Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:47.26997543Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:47.27000043Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:47.270008111Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:47.270013681Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:47.270019051Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:47.270025871Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:47.270031441Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:47.270038501Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:47.270043851Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.270149752Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"4b24202d-7c64-40d6-8d54-99468527a9da","remote_addr":"[fd00:1122:3344:101::1]:63238","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\", TimeNotSynchronized), (\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\", TimeNotSynchronized), (\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\", TimeNotSynchronized), (\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\", TimeNotSynchronized), (\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\", TimeNotSynchronized), (\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\", TimeNotSynchronized), (\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\", TimeNotSynchronized), (\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\", TimeNotSynchronized), (\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\", TimeNotSynchronized), (\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\", TimeNotSynchronized), (\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\", TimeNotSynchronized), (\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\", TimeNotSynchronized), (\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\", TimeNotSynchronized), (\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\", TimeNotSynchronized), (\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\", TimeNotSynchronized), (\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\", TimeNotSynchronized), (\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\", TimeNotSynchronized), (\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\", TimeNotSynchronized), (\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\", TimeNotSynchronized)]","latency_us":502604,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:47.270371875Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"252.060974ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4b24202d-7c64-40d6-8d54-99468527a9da\", \"content-length\": \"124\", \"date\": \"Wed, 10 Jul 2024 23:54:47 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4b24202d-7c64-40d6-8d54-99468527a9da\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 89504e37-0b08-442d-ace9-40ca97c7ca33, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5ae75945-01ab-454a-af38-a5dd1d09d270, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a390c63c-a68b-4301-b3c7-797628a3170a, kind: Service { id: 5ae75945-01ab-454a-af38-a5dd1d09d270 }, name: Name(\"ntp-5ae75945-01ab-454a-af38-a5dd1d09d270\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 108])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac507aba-8f76-47c6-bd22-6dff420215b2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2811064c-b3b2-438e-8e0c-aa9a3bfa436c, kind: Service { id: ac507aba-8f76-47c6-bd22-6dff420215b2 }, name: Name(\"external-dns-ac507aba-8f76-47c6-bd22-6dff420215b2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 139])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b39a7e8e-2dea-45da-9a7a-6977886585dc, kind: Service { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa }, name: Name(\"external-dns-b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 77])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: c108631e-f757-4a42-b510-28307153f426, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6ab9e3b5-f385-4e7e-9bc6-6c4f7a443782, kind: Service { id: c108631e-f757-4a42-b510-28307153f426 }, name: Name(\"nexus-c108631e-f757-4a42-b510-28307153f426\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 191])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6caaab71-9cdd-4faa-b082-987c9dd43604, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: ae27ee1d-0f9e-4fca-b104-9220b85a9848, kind: Service { id: 6caaab71-9cdd-4faa-b082-987c9dd43604 }, name: Name(\"nexus-6caaab71-9cdd-4faa-b082-987c9dd43604\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 147])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 37d6282c-2511-4c34-b419-e68b280bef44, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 97294434-1769-4a97-a97a-20d887e8c50a, kind: Service { id: 37d6282c-2511-4c34-b419-e68b280bef44 }, name: Name(\"nexus-37d6282c-2511-4c34-b419-e68b280bef44\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 88])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: abb1e589-7222-4a55-90ee-b0be940b410b, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: f864e092-4084-4307-8b83-2fd99a97943d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: fce1706b-b90a-4a5d-86b7-20320feb3342, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 32f00476-bb7b-4627-82ad-33c437d3a384, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: e736985c-90fa-417e-89f0-76ea8e60041f, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5baef1a-4fec-4af8-b323-da15f9e7b116, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 21592584-d863-4125-9acf-64a36ff982d8, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.523953879Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.524291149Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.524299624Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.549330608Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.549378768Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.571367618Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.571433158Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.591490974Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.591532786Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.611779977Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.611826878Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.632754345Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.632810386Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.65163252Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.651684131Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.67277011Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.672819351Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.692530055Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.692581396Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.714614175Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.714663536Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.736135899Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.73618261Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.755982575Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.756033506Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.774438766Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.774480896Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.79689093Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.79693888Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.820080742Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.820132093Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.841576876Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.841632306Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.862909288Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.862968958Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.882699613Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.882747724Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.904267567Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.904315488Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.9238227Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.923871101Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.946173493Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Reading ledger from /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.946227324Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.946495507Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:47.946645118Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:48.04616581Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:48.046233461Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:48.046241851Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:48.046247741Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:48.046253542Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:48.046259071Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:48.046264631Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:48.046270412Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:48.046276072Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:48.046281562Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:48.046287132Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:48.046292392Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:48.046298142Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:48.046305512Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:48.046312472Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:48.046337012Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:48.046342872Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:48.046348432Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:48.046353712Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:48.046457474Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"ac4be4f7-68cf-4050-836c-8b8b78b61e45","remote_addr":"[fd00:1122:3344:101::1]:63238","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\", TimeNotSynchronized), (\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\", TimeNotSynchronized), (\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\", TimeNotSynchronized), (\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\", TimeNotSynchronized), (\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\", TimeNotSynchronized), (\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\", TimeNotSynchronized), (\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\", TimeNotSynchronized), (\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\", TimeNotSynchronized), (\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\", TimeNotSynchronized), (\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\", TimeNotSynchronized), (\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\", TimeNotSynchronized), (\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\", TimeNotSynchronized), (\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\", TimeNotSynchronized), (\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\", TimeNotSynchronized), (\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\", TimeNotSynchronized), (\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\", TimeNotSynchronized), (\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\", TimeNotSynchronized), (\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\", TimeNotSynchronized), (\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\", TimeNotSynchronized)]","latency_us":522322,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:48.046700856Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"205.399328ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ac4be4f7-68cf-4050-836c-8b8b78b61e45\", \"content-length\": \"124\", \"date\": \"Wed, 10 Jul 2024 23:54:48 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ac4be4f7-68cf-4050-836c-8b8b78b61e45\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 89504e37-0b08-442d-ace9-40ca97c7ca33, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5ae75945-01ab-454a-af38-a5dd1d09d270, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a390c63c-a68b-4301-b3c7-797628a3170a, kind: Service { id: 5ae75945-01ab-454a-af38-a5dd1d09d270 }, name: Name(\"ntp-5ae75945-01ab-454a-af38-a5dd1d09d270\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 108])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac507aba-8f76-47c6-bd22-6dff420215b2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2811064c-b3b2-438e-8e0c-aa9a3bfa436c, kind: Service { id: ac507aba-8f76-47c6-bd22-6dff420215b2 }, name: Name(\"external-dns-ac507aba-8f76-47c6-bd22-6dff420215b2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 139])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b39a7e8e-2dea-45da-9a7a-6977886585dc, kind: Service { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa }, name: Name(\"external-dns-b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 77])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: c108631e-f757-4a42-b510-28307153f426, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6ab9e3b5-f385-4e7e-9bc6-6c4f7a443782, kind: Service { id: c108631e-f757-4a42-b510-28307153f426 }, name: Name(\"nexus-c108631e-f757-4a42-b510-28307153f426\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 191])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6caaab71-9cdd-4faa-b082-987c9dd43604, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: ae27ee1d-0f9e-4fca-b104-9220b85a9848, kind: Service { id: 6caaab71-9cdd-4faa-b082-987c9dd43604 }, name: Name(\"nexus-6caaab71-9cdd-4faa-b082-987c9dd43604\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 147])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 37d6282c-2511-4c34-b419-e68b280bef44, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 97294434-1769-4a97-a97a-20d887e8c50a, kind: Service { id: 37d6282c-2511-4c34-b419-e68b280bef44 }, name: Name(\"nexus-37d6282c-2511-4c34-b419-e68b280bef44\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 88])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: abb1e589-7222-4a55-90ee-b0be940b410b, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: f864e092-4084-4307-8b83-2fd99a97943d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: fce1706b-b90a-4a5d-86b7-20320feb3342, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 32f00476-bb7b-4627-82ad-33c437d3a384, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: e736985c-90fa-417e-89f0-76ea8e60041f, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5baef1a-4fec-4af8-b323-da15f9e7b116, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 21592584-d863-4125-9acf-64a36ff982d8, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:48.253223792Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:48.253541696Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:48.253552596Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:48.274408032Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:48.274454033Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:48.295943406Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:48.295991967Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:48.316693722Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:48.316747793Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:48.336911672Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:48.336953263Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:48.355343243Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:48.355390333Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:48.373849014Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:48.373892184Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:48.392271164Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:48.392312985Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:48.412225431Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:48.412286172Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:48.434483913Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:48.434528773Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:48.456411462Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:48.456460352Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:48.476662322Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:48.476707222Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:48.496876742Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:48.496919152Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:48.515922749Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:48.515962629Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:48.539640207Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:48.539687137Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:48.562414474Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:48.562458104Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:48.584039939Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:48.58408326Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:48.604209369Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:48.604264789Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:48.625704142Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:48.625748533Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:48.646732231Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:48.646775381Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:48.668307206Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Reading ledger from /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:48.668367336Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:48.668539088Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:48.66869869Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:48.772248176Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:48.772313657Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:48.772320686Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:48.772326477Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:48.772341607Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:48.772348817Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:48.772354727Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:48.772361557Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:48.772367067Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:48.772372637Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:48.772377957Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:48.772383277Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:48.772390327Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:48.772395417Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:48.772400887Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:48.772407647Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:48.772413227Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:48.772418428Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:48.772425068Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:48.772528179Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"9b086518-1f16-4a34-904d-fae6aef6aefc","remote_addr":"[fd00:1122:3344:101::1]:63238","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\", TimeNotSynchronized), (\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\", TimeNotSynchronized), (\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\", TimeNotSynchronized), (\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\", TimeNotSynchronized), (\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\", TimeNotSynchronized), (\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\", TimeNotSynchronized), (\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\", TimeNotSynchronized), (\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\", TimeNotSynchronized), (\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\", TimeNotSynchronized), (\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\", TimeNotSynchronized), (\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\", TimeNotSynchronized), (\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\", TimeNotSynchronized), (\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\", TimeNotSynchronized), (\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\", TimeNotSynchronized), (\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\", TimeNotSynchronized), (\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\", TimeNotSynchronized), (\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\", TimeNotSynchronized), (\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\", TimeNotSynchronized), (\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\", TimeNotSynchronized)]","latency_us":519167,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:48.772783302Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"388.903348ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9b086518-1f16-4a34-904d-fae6aef6aefc\", \"content-length\": \"124\", \"date\": \"Wed, 10 Jul 2024 23:54:48 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9b086518-1f16-4a34-904d-fae6aef6aefc\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 89504e37-0b08-442d-ace9-40ca97c7ca33, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5ae75945-01ab-454a-af38-a5dd1d09d270, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a390c63c-a68b-4301-b3c7-797628a3170a, kind: Service { id: 5ae75945-01ab-454a-af38-a5dd1d09d270 }, name: Name(\"ntp-5ae75945-01ab-454a-af38-a5dd1d09d270\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 108])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac507aba-8f76-47c6-bd22-6dff420215b2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2811064c-b3b2-438e-8e0c-aa9a3bfa436c, kind: Service { id: ac507aba-8f76-47c6-bd22-6dff420215b2 }, name: Name(\"external-dns-ac507aba-8f76-47c6-bd22-6dff420215b2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 139])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b39a7e8e-2dea-45da-9a7a-6977886585dc, kind: Service { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa }, name: Name(\"external-dns-b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 77])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: c108631e-f757-4a42-b510-28307153f426, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6ab9e3b5-f385-4e7e-9bc6-6c4f7a443782, kind: Service { id: c108631e-f757-4a42-b510-28307153f426 }, name: Name(\"nexus-c108631e-f757-4a42-b510-28307153f426\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 191])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6caaab71-9cdd-4faa-b082-987c9dd43604, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: ae27ee1d-0f9e-4fca-b104-9220b85a9848, kind: Service { id: 6caaab71-9cdd-4faa-b082-987c9dd43604 }, name: Name(\"nexus-6caaab71-9cdd-4faa-b082-987c9dd43604\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 147])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 37d6282c-2511-4c34-b419-e68b280bef44, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 97294434-1769-4a97-a97a-20d887e8c50a, kind: Service { id: 37d6282c-2511-4c34-b419-e68b280bef44 }, name: Name(\"nexus-37d6282c-2511-4c34-b419-e68b280bef44\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 88])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: abb1e589-7222-4a55-90ee-b0be940b410b, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: f864e092-4084-4307-8b83-2fd99a97943d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: fce1706b-b90a-4a5d-86b7-20320feb3342, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 32f00476-bb7b-4627-82ad-33c437d3a384, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: e736985c-90fa-417e-89f0-76ea8e60041f, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5baef1a-4fec-4af8-b323-da15f9e7b116, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 21592584-d863-4125-9acf-64a36ff982d8, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:49.163189227Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:49.16351636Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:49.16353122Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:49.18651Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:49.186558091Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:49.206945283Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:49.206999093Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:49.233889205Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:49.233935216Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:49.255704812Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:49.255754553Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:49.276392507Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:49.276437658Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:49.297116362Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:49.297158403Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:49.318159471Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:49.318204642Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:49.3401723Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:49.340216521Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:49.361467132Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:49.361521073Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:49.382908825Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:49.382950596Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:49.403057024Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:49.403099395Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:49.424211064Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:49.424254995Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:49.444523225Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:49.444564446Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:49.463849305Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:49.463892946Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:49.485777594Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:49.485820134Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:49.506849883Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:49.506896284Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:49.527119713Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:49.527167294Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:49.547881279Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:49.5479321Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:49.571478886Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:49.571526106Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:49.591355422Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Reading ledger from /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:49.591414172Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:49.591609694Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:49.591730256Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:49.694051488Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:49.694120909Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:49.694128559Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:49.694138169Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:49.69414718Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:49.69415693Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:49.69416642Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:49.69417428Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:49.69419584Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:49.6942022Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:49.69421009Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:49.69421644Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:49.69422446Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:49.694232681Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:49.694242301Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:49.694252181Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:49.694258321Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:49.694270611Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:49.694278571Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:49.694332451Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"bdf249c4-be1b-4f48-8e53-ce900662354f","remote_addr":"[fd00:1122:3344:101::1]:63238","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\", TimeNotSynchronized), (\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\", TimeNotSynchronized), (\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\", TimeNotSynchronized), (\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\", TimeNotSynchronized), (\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\", TimeNotSynchronized), (\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\", TimeNotSynchronized), (\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\", TimeNotSynchronized), (\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\", TimeNotSynchronized), (\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\", TimeNotSynchronized), (\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\", TimeNotSynchronized), (\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\", TimeNotSynchronized), (\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\", TimeNotSynchronized), (\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\", TimeNotSynchronized), (\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\", TimeNotSynchronized), (\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\", TimeNotSynchronized), (\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\", TimeNotSynchronized), (\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\", TimeNotSynchronized), (\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\", TimeNotSynchronized), (\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\", TimeNotSynchronized)]","latency_us":530967,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:49.694612255Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"372.047195ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bdf249c4-be1b-4f48-8e53-ce900662354f\", \"content-length\": \"124\", \"date\": \"Wed, 10 Jul 2024 23:54:49 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"bdf249c4-be1b-4f48-8e53-ce900662354f\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 89504e37-0b08-442d-ace9-40ca97c7ca33, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5ae75945-01ab-454a-af38-a5dd1d09d270, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a390c63c-a68b-4301-b3c7-797628a3170a, kind: Service { id: 5ae75945-01ab-454a-af38-a5dd1d09d270 }, name: Name(\"ntp-5ae75945-01ab-454a-af38-a5dd1d09d270\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 108])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac507aba-8f76-47c6-bd22-6dff420215b2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2811064c-b3b2-438e-8e0c-aa9a3bfa436c, kind: Service { id: ac507aba-8f76-47c6-bd22-6dff420215b2 }, name: Name(\"external-dns-ac507aba-8f76-47c6-bd22-6dff420215b2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 139])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b39a7e8e-2dea-45da-9a7a-6977886585dc, kind: Service { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa }, name: Name(\"external-dns-b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 77])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: c108631e-f757-4a42-b510-28307153f426, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6ab9e3b5-f385-4e7e-9bc6-6c4f7a443782, kind: Service { id: c108631e-f757-4a42-b510-28307153f426 }, name: Name(\"nexus-c108631e-f757-4a42-b510-28307153f426\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 191])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6caaab71-9cdd-4faa-b082-987c9dd43604, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: ae27ee1d-0f9e-4fca-b104-9220b85a9848, kind: Service { id: 6caaab71-9cdd-4faa-b082-987c9dd43604 }, name: Name(\"nexus-6caaab71-9cdd-4faa-b082-987c9dd43604\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 147])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 37d6282c-2511-4c34-b419-e68b280bef44, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 97294434-1769-4a97-a97a-20d887e8c50a, kind: Service { id: 37d6282c-2511-4c34-b419-e68b280bef44 }, name: Name(\"nexus-37d6282c-2511-4c34-b419-e68b280bef44\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 88])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: abb1e589-7222-4a55-90ee-b0be940b410b, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: f864e092-4084-4307-8b83-2fd99a97943d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: fce1706b-b90a-4a5d-86b7-20320feb3342, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 32f00476-bb7b-4627-82ad-33c437d3a384, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: e736985c-90fa-417e-89f0-76ea8e60041f, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5baef1a-4fec-4af8-b323-da15f9e7b116, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 21592584-d863-4125-9acf-64a36ff982d8, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.068194927Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.0685274Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.068534691Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.092665653Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.092730624Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.115614832Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.115659803Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.137313068Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.137365139Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.157915312Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.157959383Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.177814939Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.177860869Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.198428843Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.198471993Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.218108247Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.218148558Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.240088656Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.240143357Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.26075309Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.260799101Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.283532028Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.283590939Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.303087231Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.303138271Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.324044679Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.324090429Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.344975946Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.345027607Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.364970014Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.365014994Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.387365977Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.387420898Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.408615208Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.408663529Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.430197353Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.430250114Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.452503806Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.452560386Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.474532575Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.474577356Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.495998719Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Reading ledger from /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.496063849Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.496267431Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.496410373Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:50.600387172Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:50.600448073Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:50.600456773Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:50.600464526Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:50.600472026Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:50.600478195Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:50.600485695Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:50.600491714Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:50.600495989Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:50.600501952Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:50.600506265Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:50.600510446Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:50.600528371Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:50.600534212Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:50.600539883Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:50.60054413Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:50.600548312Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:50.600554077Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:50.600559852Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.600604009Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"e2184b64-2c1b-4b52-9b1a-633a7cb9a46d","remote_addr":"[fd00:1122:3344:101::1]:63238","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\", TimeNotSynchronized), (\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\", TimeNotSynchronized), (\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\", TimeNotSynchronized), (\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\", TimeNotSynchronized), (\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\", TimeNotSynchronized), (\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\", TimeNotSynchronized), (\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\", TimeNotSynchronized), (\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\", TimeNotSynchronized), (\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\", TimeNotSynchronized), (\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\", TimeNotSynchronized), (\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\", TimeNotSynchronized), (\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\", TimeNotSynchronized), (\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\", TimeNotSynchronized), (\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\", TimeNotSynchronized), (\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\", TimeNotSynchronized), (\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\", TimeNotSynchronized), (\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\", TimeNotSynchronized), (\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\", TimeNotSynchronized), (\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\", TimeNotSynchronized)]","latency_us":532301,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:50.600873999Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"358.724667ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e2184b64-2c1b-4b52-9b1a-633a7cb9a46d\", \"content-length\": \"124\", \"date\": \"Wed, 10 Jul 2024 23:54:49 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e2184b64-2c1b-4b52-9b1a-633a7cb9a46d\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 89504e37-0b08-442d-ace9-40ca97c7ca33, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5ae75945-01ab-454a-af38-a5dd1d09d270, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a390c63c-a68b-4301-b3c7-797628a3170a, kind: Service { id: 5ae75945-01ab-454a-af38-a5dd1d09d270 }, name: Name(\"ntp-5ae75945-01ab-454a-af38-a5dd1d09d270\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 108])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac507aba-8f76-47c6-bd22-6dff420215b2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2811064c-b3b2-438e-8e0c-aa9a3bfa436c, kind: Service { id: ac507aba-8f76-47c6-bd22-6dff420215b2 }, name: Name(\"external-dns-ac507aba-8f76-47c6-bd22-6dff420215b2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 139])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b39a7e8e-2dea-45da-9a7a-6977886585dc, kind: Service { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa }, name: Name(\"external-dns-b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 77])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: c108631e-f757-4a42-b510-28307153f426, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6ab9e3b5-f385-4e7e-9bc6-6c4f7a443782, kind: Service { id: c108631e-f757-4a42-b510-28307153f426 }, name: Name(\"nexus-c108631e-f757-4a42-b510-28307153f426\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 191])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6caaab71-9cdd-4faa-b082-987c9dd43604, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: ae27ee1d-0f9e-4fca-b104-9220b85a9848, kind: Service { id: 6caaab71-9cdd-4faa-b082-987c9dd43604 }, name: Name(\"nexus-6caaab71-9cdd-4faa-b082-987c9dd43604\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 147])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 37d6282c-2511-4c34-b419-e68b280bef44, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 97294434-1769-4a97-a97a-20d887e8c50a, kind: Service { id: 37d6282c-2511-4c34-b419-e68b280bef44 }, name: Name(\"nexus-37d6282c-2511-4c34-b419-e68b280bef44\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 88])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: abb1e589-7222-4a55-90ee-b0be940b410b, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: f864e092-4084-4307-8b83-2fd99a97943d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: fce1706b-b90a-4a5d-86b7-20320feb3342, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 32f00476-bb7b-4627-82ad-33c437d3a384, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: e736985c-90fa-417e-89f0-76ea8e60041f, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5baef1a-4fec-4af8-b323-da15f9e7b116, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 21592584-d863-4125-9acf-64a36ff982d8, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.96150138Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.961883554Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.961894205Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.986224549Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:50.98627817Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:51.005514899Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:51.00557791Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:51.028361517Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:51.028409128Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:51.047798139Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:51.047846499Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:51.066645834Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:51.066685464Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:51.087034195Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:51.087073536Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:51.107863072Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:51.107932383Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:51.129453207Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:51.129503227Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:51.151412015Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:51.151457416Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:51.172660566Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:51.172700627Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:51.196705308Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:51.196758968Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:51.218716597Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:51.218764228Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:51.240287392Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:51.240339252Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:51.260528452Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:51.260569172Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:51.283280479Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:51.2833289Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:51.303518539Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:51.30357294Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:51.323995952Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:51.324040132Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:51.347678429Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:51.34772927Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:51.368137892Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:51.368187372Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:51.388046668Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Reading ledger from /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:51.388095639Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:51.388307821Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:51.388424943Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:51.489486411Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:51.489545262Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:51.489553632Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:51.489560942Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:51.489565072Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:51.489570512Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:51.489576123Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:51.489582953Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:51.489588353Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:51.489595093Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:51.489600673Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:51.489606233Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:51.489613343Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:51.489620383Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:51.489626123Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:51.489640043Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:51.489645433Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:51.489649123Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:51.489652613Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:51.489720434Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"1a3dfc10-7a18-4b5b-92f6-7c63f84b582e","remote_addr":"[fd00:1122:3344:101::1]:63238","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\", TimeNotSynchronized), (\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\", TimeNotSynchronized), (\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\", TimeNotSynchronized), (\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\", TimeNotSynchronized), (\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\", TimeNotSynchronized), (\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\", TimeNotSynchronized), (\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\", TimeNotSynchronized), (\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\", TimeNotSynchronized), (\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\", TimeNotSynchronized), (\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\", TimeNotSynchronized), (\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\", TimeNotSynchronized), (\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\", TimeNotSynchronized), (\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\", TimeNotSynchronized), (\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\", TimeNotSynchronized), (\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\", TimeNotSynchronized), (\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\", TimeNotSynchronized), (\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\", TimeNotSynchronized), (\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\", TimeNotSynchronized), (\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\", TimeNotSynchronized)]","latency_us":528061,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:51.489897656Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"495.227288ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1a3dfc10-7a18-4b5b-92f6-7c63f84b582e\", \"content-length\": \"124\", \"date\": \"Wed, 10 Jul 2024 23:54:50 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1a3dfc10-7a18-4b5b-92f6-7c63f84b582e\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 89504e37-0b08-442d-ace9-40ca97c7ca33, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5ae75945-01ab-454a-af38-a5dd1d09d270, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a390c63c-a68b-4301-b3c7-797628a3170a, kind: Service { id: 5ae75945-01ab-454a-af38-a5dd1d09d270 }, name: Name(\"ntp-5ae75945-01ab-454a-af38-a5dd1d09d270\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 108])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac507aba-8f76-47c6-bd22-6dff420215b2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2811064c-b3b2-438e-8e0c-aa9a3bfa436c, kind: Service { id: ac507aba-8f76-47c6-bd22-6dff420215b2 }, name: Name(\"external-dns-ac507aba-8f76-47c6-bd22-6dff420215b2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 139])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b39a7e8e-2dea-45da-9a7a-6977886585dc, kind: Service { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa }, name: Name(\"external-dns-b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 77])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: c108631e-f757-4a42-b510-28307153f426, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6ab9e3b5-f385-4e7e-9bc6-6c4f7a443782, kind: Service { id: c108631e-f757-4a42-b510-28307153f426 }, name: Name(\"nexus-c108631e-f757-4a42-b510-28307153f426\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 191])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6caaab71-9cdd-4faa-b082-987c9dd43604, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: ae27ee1d-0f9e-4fca-b104-9220b85a9848, kind: Service { id: 6caaab71-9cdd-4faa-b082-987c9dd43604 }, name: Name(\"nexus-6caaab71-9cdd-4faa-b082-987c9dd43604\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 147])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 37d6282c-2511-4c34-b419-e68b280bef44, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 97294434-1769-4a97-a97a-20d887e8c50a, kind: Service { id: 37d6282c-2511-4c34-b419-e68b280bef44 }, name: Name(\"nexus-37d6282c-2511-4c34-b419-e68b280bef44\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 88])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: abb1e589-7222-4a55-90ee-b0be940b410b, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: f864e092-4084-4307-8b83-2fd99a97943d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: fce1706b-b90a-4a5d-86b7-20320feb3342, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 32f00476-bb7b-4627-82ad-33c437d3a384, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: e736985c-90fa-417e-89f0-76ea8e60041f, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5baef1a-4fec-4af8-b323-da15f9e7b116, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 21592584-d863-4125-9acf-64a36ff982d8, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:51.986533966Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:51.98686159Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:51.98687134Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:52.008285893Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:52.008338693Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:52.033336575Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:52.033391056Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:52.056229324Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:52.056281535Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:52.078061922Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:52.078112632Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:52.100350474Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:52.100399785Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:52.124591538Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:52.124641888Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:52.147393025Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:52.147440926Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:52.169069761Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:52.169127892Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:52.191337743Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:52.191394844Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:52.213645776Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:52.213695877Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:52.238027651Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:52.238082932Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:52.263486898Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:52.263536008Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:52.288568651Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:52.288627711Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:52.310767732Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:52.310820993Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:52.334421429Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:52.33447725Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:52.361624395Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:52.361678516Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:52.386127861Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:52.386174942Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:52.40986695Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:52.40992283Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:52.43288783Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:52.432946121Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:52.454131731Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Reading ledger from /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:52.454190372Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:52.454370203Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:52.454516545Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:52.558159502Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:52.558217513Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:52.558225123Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:52.558230803Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:52.558257473Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:52.558262913Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:52.558266623Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:52.558270483Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:52.558275573Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:52.558279444Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:52.558283024Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:52.558288173Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:52.558292013Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:52.558297244Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:52.558302384Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:52.558307234Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:52.558311374Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:52.558318774Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:52.558325234Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:52.558349384Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"1128b967-e189-4ee3-830d-d283e96f413a","remote_addr":"[fd00:1122:3344:101::1]:63238","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\", TimeNotSynchronized), (\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\", TimeNotSynchronized), (\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\", TimeNotSynchronized), (\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\", TimeNotSynchronized), (\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\", TimeNotSynchronized), (\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\", TimeNotSynchronized), (\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\", TimeNotSynchronized), (\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\", TimeNotSynchronized), (\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\", TimeNotSynchronized), (\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\", TimeNotSynchronized), (\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\", TimeNotSynchronized), (\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\", TimeNotSynchronized), (\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\", TimeNotSynchronized), (\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\", TimeNotSynchronized), (\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\", TimeNotSynchronized), (\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\", TimeNotSynchronized), (\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\", TimeNotSynchronized), (\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\", TimeNotSynchronized), (\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\", TimeNotSynchronized)]","latency_us":571682,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:52.558534296Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"635.865341ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1128b967-e189-4ee3-830d-d283e96f413a\", \"content-length\": \"124\", \"date\": \"Wed, 10 Jul 2024 23:54:51 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1128b967-e189-4ee3-830d-d283e96f413a\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 89504e37-0b08-442d-ace9-40ca97c7ca33, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5ae75945-01ab-454a-af38-a5dd1d09d270, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a390c63c-a68b-4301-b3c7-797628a3170a, kind: Service { id: 5ae75945-01ab-454a-af38-a5dd1d09d270 }, name: Name(\"ntp-5ae75945-01ab-454a-af38-a5dd1d09d270\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 108])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac507aba-8f76-47c6-bd22-6dff420215b2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2811064c-b3b2-438e-8e0c-aa9a3bfa436c, kind: Service { id: ac507aba-8f76-47c6-bd22-6dff420215b2 }, name: Name(\"external-dns-ac507aba-8f76-47c6-bd22-6dff420215b2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 139])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b39a7e8e-2dea-45da-9a7a-6977886585dc, kind: Service { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa }, name: Name(\"external-dns-b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 77])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: c108631e-f757-4a42-b510-28307153f426, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6ab9e3b5-f385-4e7e-9bc6-6c4f7a443782, kind: Service { id: c108631e-f757-4a42-b510-28307153f426 }, name: Name(\"nexus-c108631e-f757-4a42-b510-28307153f426\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 191])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6caaab71-9cdd-4faa-b082-987c9dd43604, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: ae27ee1d-0f9e-4fca-b104-9220b85a9848, kind: Service { id: 6caaab71-9cdd-4faa-b082-987c9dd43604 }, name: Name(\"nexus-6caaab71-9cdd-4faa-b082-987c9dd43604\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 147])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 37d6282c-2511-4c34-b419-e68b280bef44, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 97294434-1769-4a97-a97a-20d887e8c50a, kind: Service { id: 37d6282c-2511-4c34-b419-e68b280bef44 }, name: Name(\"nexus-37d6282c-2511-4c34-b419-e68b280bef44\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 88])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: abb1e589-7222-4a55-90ee-b0be940b410b, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: f864e092-4084-4307-8b83-2fd99a97943d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: fce1706b-b90a-4a5d-86b7-20320feb3342, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 32f00476-bb7b-4627-82ad-33c437d3a384, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: e736985c-90fa-417e-89f0-76ea8e60041f, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5baef1a-4fec-4af8-b323-da15f9e7b116, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 21592584-d863-4125-9acf-64a36ff982d8, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:53.196493863Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:53.196843067Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:53.196855347Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:53.22014711Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:53.220202921Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:53.242532494Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:53.242582874Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:53.263119367Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:53.263177838Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:53.283434488Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:53.283484268Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:53.304270695Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:53.304318365Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:53.324348213Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:53.324405623Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:53.343985896Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:53.344037527Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:53.363148315Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:53.363204235Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:53.387737672Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:53.387795813Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:53.408993441Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:53.409037251Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:53.430524577Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:53.430575648Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:53.450360053Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:53.450411253Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:53.470964117Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:53.471007228Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:53.491155067Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:53.491218407Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:53.513135726Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:53.513181806Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:53.533530027Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:53.533576798Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:53.553325253Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:53.553367913Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:53.573905906Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:53.573955397Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:53.596145228Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:53.596189689Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:53.619636964Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Reading ledger from /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:53.619700214Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:53.619901827Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:53.620028058Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:53.719946455Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:53.720007755Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:53.720015665Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:53.720021445Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:53.720028255Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:53.720033495Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:53.720037116Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:53.720042236Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:53.720060916Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:53.720066456Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:53.720071846Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:53.720076966Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:53.720082376Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:53.720086356Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:53.720090026Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:53.720094896Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:53.720099976Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:53.720103696Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:53.720108756Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:53.720126246Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"fa69e544-9244-49bc-8c2b-30e1ea26f9c6","remote_addr":"[fd00:1122:3344:101::1]:63238","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\", TimeNotSynchronized), (\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\", TimeNotSynchronized), (\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\", TimeNotSynchronized), (\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\", TimeNotSynchronized), (\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\", TimeNotSynchronized), (\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\", TimeNotSynchronized), (\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\", TimeNotSynchronized), (\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\", TimeNotSynchronized), (\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\", TimeNotSynchronized), (\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\", TimeNotSynchronized), (\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\", TimeNotSynchronized), (\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\", TimeNotSynchronized), (\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\", TimeNotSynchronized), (\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\", TimeNotSynchronized), (\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\", TimeNotSynchronized), (\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\", TimeNotSynchronized), (\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\", TimeNotSynchronized), (\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\", TimeNotSynchronized), (\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\", TimeNotSynchronized)]","latency_us":523496,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:53.720317468Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"773.014035ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fa69e544-9244-49bc-8c2b-30e1ea26f9c6\", \"content-length\": \"124\", \"date\": \"Wed, 10 Jul 2024 23:54:53 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"fa69e544-9244-49bc-8c2b-30e1ea26f9c6\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 89504e37-0b08-442d-ace9-40ca97c7ca33, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5ae75945-01ab-454a-af38-a5dd1d09d270, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a390c63c-a68b-4301-b3c7-797628a3170a, kind: Service { id: 5ae75945-01ab-454a-af38-a5dd1d09d270 }, name: Name(\"ntp-5ae75945-01ab-454a-af38-a5dd1d09d270\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 108])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac507aba-8f76-47c6-bd22-6dff420215b2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2811064c-b3b2-438e-8e0c-aa9a3bfa436c, kind: Service { id: ac507aba-8f76-47c6-bd22-6dff420215b2 }, name: Name(\"external-dns-ac507aba-8f76-47c6-bd22-6dff420215b2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 139])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b39a7e8e-2dea-45da-9a7a-6977886585dc, kind: Service { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa }, name: Name(\"external-dns-b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 77])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: c108631e-f757-4a42-b510-28307153f426, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6ab9e3b5-f385-4e7e-9bc6-6c4f7a443782, kind: Service { id: c108631e-f757-4a42-b510-28307153f426 }, name: Name(\"nexus-c108631e-f757-4a42-b510-28307153f426\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 191])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6caaab71-9cdd-4faa-b082-987c9dd43604, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: ae27ee1d-0f9e-4fca-b104-9220b85a9848, kind: Service { id: 6caaab71-9cdd-4faa-b082-987c9dd43604 }, name: Name(\"nexus-6caaab71-9cdd-4faa-b082-987c9dd43604\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 147])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 37d6282c-2511-4c34-b419-e68b280bef44, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 97294434-1769-4a97-a97a-20d887e8c50a, kind: Service { id: 37d6282c-2511-4c34-b419-e68b280bef44 }, name: Name(\"nexus-37d6282c-2511-4c34-b419-e68b280bef44\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 88])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: abb1e589-7222-4a55-90ee-b0be940b410b, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: f864e092-4084-4307-8b83-2fd99a97943d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: fce1706b-b90a-4a5d-86b7-20320feb3342, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 32f00476-bb7b-4627-82ad-33c437d3a384, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: e736985c-90fa-417e-89f0-76ea8e60041f, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5baef1a-4fec-4af8-b323-da15f9e7b116, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 21592584-d863-4125-9acf-64a36ff982d8, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:54.49383923Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:54.494163083Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:54.494173903Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:54.515685537Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:54.515743778Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:54.534220969Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:54.534265159Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:54.556296598Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:54.556347419Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:54.575028802Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:54.575080233Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:54.594532484Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:54.594581205Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:54.614086307Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:54.614132238Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:54.634014304Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:54.634064384Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:54.654362885Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:54.654414545Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:54.673485183Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:54.673533803Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:54.692657231Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:54.692697782Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:54.71088885Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:54.71092834Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:54.728939136Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:54.728992116Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:54.749397728Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:54.749445509Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:54.768098812Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:54.768143552Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:54.788075819Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:54.78813693Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:54.806431829Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:54.806471669Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:54.824745938Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:54.824790268Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:54.846476614Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:54.846526614Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:54.866623903Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:54.866662764Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:54.886089915Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Reading ledger from /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:54.886136245Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:54.886319007Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:54.886452199Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:54.984261422Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:54.984323183Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:54.984331643Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:54.984337533Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:54.984341583Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:54.984348473Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:54.984353863Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:54.984359183Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:54.984364433Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:54.984371373Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:54.984374984Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:54.984378764Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:54.984398804Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:54.984404254Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:54.984409674Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:54.984415324Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:54.984420304Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:54.984425694Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:54.984429374Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:54.984481285Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"8afe8cdb-20cd-40cc-8c7c-8662ba410178","remote_addr":"[fd00:1122:3344:101::1]:63238","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\", TimeNotSynchronized), (\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\", TimeNotSynchronized), (\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\", TimeNotSynchronized), (\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\", TimeNotSynchronized), (\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\", TimeNotSynchronized), (\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\", TimeNotSynchronized), (\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\", TimeNotSynchronized), (\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\", TimeNotSynchronized), (\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\", TimeNotSynchronized), (\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\", TimeNotSynchronized), (\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\", TimeNotSynchronized), (\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\", TimeNotSynchronized), (\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\", TimeNotSynchronized), (\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\", TimeNotSynchronized), (\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\", TimeNotSynchronized), (\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\", TimeNotSynchronized), (\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\", TimeNotSynchronized), (\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\", TimeNotSynchronized), (\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\", TimeNotSynchronized)]","latency_us":490503,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:54.984690827Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"507.489325ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8afe8cdb-20cd-40cc-8c7c-8662ba410178\", \"content-length\": \"124\", \"date\": \"Wed, 10 Jul 2024 23:54:54 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8afe8cdb-20cd-40cc-8c7c-8662ba410178\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 89504e37-0b08-442d-ace9-40ca97c7ca33, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5ae75945-01ab-454a-af38-a5dd1d09d270, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a390c63c-a68b-4301-b3c7-797628a3170a, kind: Service { id: 5ae75945-01ab-454a-af38-a5dd1d09d270 }, name: Name(\"ntp-5ae75945-01ab-454a-af38-a5dd1d09d270\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 108])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac507aba-8f76-47c6-bd22-6dff420215b2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2811064c-b3b2-438e-8e0c-aa9a3bfa436c, kind: Service { id: ac507aba-8f76-47c6-bd22-6dff420215b2 }, name: Name(\"external-dns-ac507aba-8f76-47c6-bd22-6dff420215b2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 139])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b39a7e8e-2dea-45da-9a7a-6977886585dc, kind: Service { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa }, name: Name(\"external-dns-b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 77])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: c108631e-f757-4a42-b510-28307153f426, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6ab9e3b5-f385-4e7e-9bc6-6c4f7a443782, kind: Service { id: c108631e-f757-4a42-b510-28307153f426 }, name: Name(\"nexus-c108631e-f757-4a42-b510-28307153f426\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 191])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6caaab71-9cdd-4faa-b082-987c9dd43604, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: ae27ee1d-0f9e-4fca-b104-9220b85a9848, kind: Service { id: 6caaab71-9cdd-4faa-b082-987c9dd43604 }, name: Name(\"nexus-6caaab71-9cdd-4faa-b082-987c9dd43604\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 147])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 37d6282c-2511-4c34-b419-e68b280bef44, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 97294434-1769-4a97-a97a-20d887e8c50a, kind: Service { id: 37d6282c-2511-4c34-b419-e68b280bef44 }, name: Name(\"nexus-37d6282c-2511-4c34-b419-e68b280bef44\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 88])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: abb1e589-7222-4a55-90ee-b0be940b410b, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: f864e092-4084-4307-8b83-2fd99a97943d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: fce1706b-b90a-4a5d-86b7-20320feb3342, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 32f00476-bb7b-4627-82ad-33c437d3a384, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: e736985c-90fa-417e-89f0-76ea8e60041f, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5baef1a-4fec-4af8-b323-da15f9e7b116, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 21592584-d863-4125-9acf-64a36ff982d8, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:55.493307587Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:55.493631811Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:55.493640291Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:55.515660041Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:55.515707291Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:55.537365886Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:55.537419797Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:55.559177014Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:55.559219104Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:55.578902188Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:55.578946849Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:55.601602715Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:55.601646525Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:55.622707125Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:55.622758315Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:55.642813193Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:55.642877734Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:55.662408326Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:55.662449937Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:55.68485639Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:55.684912381Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:55.705784888Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:55.705830409Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:55.725505992Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:55.725550933Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:55.746521321Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:55.746564011Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:55.766919083Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:55.766960493Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:55.787025201Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:55.787066662Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:55.809046941Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:55.809099251Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:55.832795099Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:55.83285302Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:55.854785538Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:55.854828189Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:55.876823878Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:55.876871698Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:55.897781166Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:55.897828696Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:55.918779804Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Reading ledger from /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:55.918828334Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:55.919009987Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:55.919142908Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:56.017868751Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:56.017928042Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:56.017933722Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:56.017937902Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:56.017944632Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:56.017949702Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:56.017956343Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:56.017962192Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:56.017967612Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:56.017972933Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:56.017978423Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:56.017985273Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:56.017990823Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:56.017994723Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:56.017999823Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:56.018021203Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:56.018026963Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:56.018032303Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:56.018039083Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:56.018095364Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"b2bba31f-ca2e-4fde-a9a1-87d669100bd7","remote_addr":"[fd00:1122:3344:101::1]:63238","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\", TimeNotSynchronized), (\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\", TimeNotSynchronized), (\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\", TimeNotSynchronized), (\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\", TimeNotSynchronized), (\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\", TimeNotSynchronized), (\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\", TimeNotSynchronized), (\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\", TimeNotSynchronized), (\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\", TimeNotSynchronized), (\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\", TimeNotSynchronized), (\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\", TimeNotSynchronized), (\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\", TimeNotSynchronized), (\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\", TimeNotSynchronized), (\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\", TimeNotSynchronized), (\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\", TimeNotSynchronized), (\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\", TimeNotSynchronized), (\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\", TimeNotSynchronized), (\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\", TimeNotSynchronized), (\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\", TimeNotSynchronized), (\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\", TimeNotSynchronized)]","latency_us":524648,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:56.018298356Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"1.492231156s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b2bba31f-ca2e-4fde-a9a1-87d669100bd7\", \"content-length\": \"124\", \"date\": \"Wed, 10 Jul 2024 23:54:56 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b2bba31f-ca2e-4fde-a9a1-87d669100bd7\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 89504e37-0b08-442d-ace9-40ca97c7ca33, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5ae75945-01ab-454a-af38-a5dd1d09d270, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a390c63c-a68b-4301-b3c7-797628a3170a, kind: Service { id: 5ae75945-01ab-454a-af38-a5dd1d09d270 }, name: Name(\"ntp-5ae75945-01ab-454a-af38-a5dd1d09d270\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 108])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac507aba-8f76-47c6-bd22-6dff420215b2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2811064c-b3b2-438e-8e0c-aa9a3bfa436c, kind: Service { id: ac507aba-8f76-47c6-bd22-6dff420215b2 }, name: Name(\"external-dns-ac507aba-8f76-47c6-bd22-6dff420215b2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 139])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b39a7e8e-2dea-45da-9a7a-6977886585dc, kind: Service { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa }, name: Name(\"external-dns-b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 77])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: c108631e-f757-4a42-b510-28307153f426, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6ab9e3b5-f385-4e7e-9bc6-6c4f7a443782, kind: Service { id: c108631e-f757-4a42-b510-28307153f426 }, name: Name(\"nexus-c108631e-f757-4a42-b510-28307153f426\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 191])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6caaab71-9cdd-4faa-b082-987c9dd43604, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: ae27ee1d-0f9e-4fca-b104-9220b85a9848, kind: Service { id: 6caaab71-9cdd-4faa-b082-987c9dd43604 }, name: Name(\"nexus-6caaab71-9cdd-4faa-b082-987c9dd43604\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 147])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 37d6282c-2511-4c34-b419-e68b280bef44, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 97294434-1769-4a97-a97a-20d887e8c50a, kind: Service { id: 37d6282c-2511-4c34-b419-e68b280bef44 }, name: Name(\"nexus-37d6282c-2511-4c34-b419-e68b280bef44\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 88])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: abb1e589-7222-4a55-90ee-b0be940b410b, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: f864e092-4084-4307-8b83-2fd99a97943d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: fce1706b-b90a-4a5d-86b7-20320feb3342, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 32f00476-bb7b-4627-82ad-33c437d3a384, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: e736985c-90fa-417e-89f0-76ea8e60041f, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5baef1a-4fec-4af8-b323-da15f9e7b116, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 21592584-d863-4125-9acf-64a36ff982d8, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:57.51225013Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:57.51314016Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:57.51315089Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:57.535440572Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:57.535493153Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:57.556496056Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:57.556539481Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:57.578537681Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:57.578591332Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:57.597805841Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:57.597851321Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:57.617840108Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:57.617888049Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:57.639031829Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:57.639073689Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:57.658119286Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:57.658165447Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:57.679066124Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:57.679129715Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:57.699224833Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:57.699273364Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:57.720328203Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:57.720382233Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:57.739970906Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:57.740016647Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:57.761848634Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:57.761891515Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:57.78166875Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:57.78171067Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:57.800885989Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:57.800922009Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:57.820470372Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:57.820515862Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:57.83961734Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:57.83966454Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:57.858848889Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:57.858887869Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:57.877212419Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:57.877248079Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:57.895529878Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:57.895569098Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:57.914475134Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Reading ledger from /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:57.914527274Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:57.914749037Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:57.914886698Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:58.00700284Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:58.00704426Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:58.00705152Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:58.007056511Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:58.007072731Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:58.007078111Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:58.007083491Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:58.007090041Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:58.007093681Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:58.007100341Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:58.007105571Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:58.007110781Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:58.007117341Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:58.007121131Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:58.007124781Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:58.007130091Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:58.007133821Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:58.007137772Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:58.007143241Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:58.007215012Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"dbf1b983-1ab7-4f6b-aa1c-8f2d95e8bd9f","remote_addr":"[fd00:1122:3344:101::1]:63238","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\", TimeNotSynchronized), (\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\", TimeNotSynchronized), (\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\", TimeNotSynchronized), (\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\", TimeNotSynchronized), (\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\", TimeNotSynchronized), (\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\", TimeNotSynchronized), (\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\", TimeNotSynchronized), (\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\", TimeNotSynchronized), (\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\", TimeNotSynchronized), (\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\", TimeNotSynchronized), (\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\", TimeNotSynchronized), (\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\", TimeNotSynchronized), (\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\", TimeNotSynchronized), (\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\", TimeNotSynchronized), (\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\", TimeNotSynchronized), (\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\", TimeNotSynchronized), (\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\", TimeNotSynchronized), (\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\", TimeNotSynchronized), (\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\", TimeNotSynchronized)]","latency_us":494836,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:58.007394154Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"1.862259122s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"dbf1b983-1ab7-4f6b-aa1c-8f2d95e8bd9f\", \"content-length\": \"124\", \"date\": \"Wed, 10 Jul 2024 23:54:57 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"dbf1b983-1ab7-4f6b-aa1c-8f2d95e8bd9f\" }"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:58.05944501Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55525"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:58.060816935Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"54f47570-8360-4cd9-b4c6-a42b9e06efd2","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55525","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":851,"response_code":"200"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:54:58.336165919Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:423","component":"producer-server","component":"MetricsManager","sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"oximeter/producer/src/lib.rs:423","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"1.06534563s"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 89504e37-0b08-442d-ace9-40ca97c7ca33, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5ae75945-01ab-454a-af38-a5dd1d09d270, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a390c63c-a68b-4301-b3c7-797628a3170a, kind: Service { id: 5ae75945-01ab-454a-af38-a5dd1d09d270 }, name: Name(\"ntp-5ae75945-01ab-454a-af38-a5dd1d09d270\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 108])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac507aba-8f76-47c6-bd22-6dff420215b2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2811064c-b3b2-438e-8e0c-aa9a3bfa436c, kind: Service { id: ac507aba-8f76-47c6-bd22-6dff420215b2 }, name: Name(\"external-dns-ac507aba-8f76-47c6-bd22-6dff420215b2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 139])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b39a7e8e-2dea-45da-9a7a-6977886585dc, kind: Service { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa }, name: Name(\"external-dns-b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 77])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: c108631e-f757-4a42-b510-28307153f426, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6ab9e3b5-f385-4e7e-9bc6-6c4f7a443782, kind: Service { id: c108631e-f757-4a42-b510-28307153f426 }, name: Name(\"nexus-c108631e-f757-4a42-b510-28307153f426\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 191])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6caaab71-9cdd-4faa-b082-987c9dd43604, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: ae27ee1d-0f9e-4fca-b104-9220b85a9848, kind: Service { id: 6caaab71-9cdd-4faa-b082-987c9dd43604 }, name: Name(\"nexus-6caaab71-9cdd-4faa-b082-987c9dd43604\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 147])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 37d6282c-2511-4c34-b419-e68b280bef44, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 97294434-1769-4a97-a97a-20d887e8c50a, kind: Service { id: 37d6282c-2511-4c34-b419-e68b280bef44 }, name: Name(\"nexus-37d6282c-2511-4c34-b419-e68b280bef44\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 88])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: abb1e589-7222-4a55-90ee-b0be940b410b, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: f864e092-4084-4307-8b83-2fd99a97943d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: fce1706b-b90a-4a5d-86b7-20320feb3342, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 32f00476-bb7b-4627-82ad-33c437d3a384, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: e736985c-90fa-417e-89f0-76ea8e60041f, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5baef1a-4fec-4af8-b323-da15f9e7b116, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 21592584-d863-4125-9acf-64a36ff982d8, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:59.871056208Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:59.871522924Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:59.871535774Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:59.893809116Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:59.893853896Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:59.913135506Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:59.913167836Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:59.93194733Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:59.931986721Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:59.951272551Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:59.951309421Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:59.969096545Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:59.969133185Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:59.98710255Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:54:59.987135271Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:00.005100076Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:00.005135146Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:00.023946651Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:00.023983901Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:00.041846905Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:00.041885736Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:00.060066314Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:00.060098194Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:00.078443583Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:00.078486884Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:00.096706312Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:00.096757543Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:00.114609417Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:00.114647217Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:00.133147068Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:00.133194729Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:00.150996663Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:00.151038303Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:00.169234021Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:00.169266761Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:00.187347288Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:00.187399788Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:00.205703037Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:00.205738828Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:00.224424251Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:00.224473402Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:00.242920102Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Reading ledger from /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:00.242959023Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:00.243188245Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:00.243315786Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:00.333166303Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:00.333212904Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:00.333230004Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:00.333236524Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:00.333244374Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:00.333250564Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:00.333258754Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:00.333266624Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:00.333272694Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:00.333280354Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:00.333286565Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:00.333294765Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:00.333302775Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:00.333309135Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:00.333313635Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:00.333319935Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:00.333324425Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:00.333330835Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:00.333348755Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:00.333415166Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"4bd843ff-520c-4636-8acd-48f99945babf","remote_addr":"[fd00:1122:3344:101::1]:63238","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\", TimeNotSynchronized), (\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\", TimeNotSynchronized), (\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\", TimeNotSynchronized), (\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\", TimeNotSynchronized), (\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\", TimeNotSynchronized), (\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\", TimeNotSynchronized), (\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\", TimeNotSynchronized), (\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\", TimeNotSynchronized), (\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\", TimeNotSynchronized), (\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\", TimeNotSynchronized), (\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\", TimeNotSynchronized), (\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\", TimeNotSynchronized), (\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\", TimeNotSynchronized), (\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\", TimeNotSynchronized), (\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\", TimeNotSynchronized), (\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\", TimeNotSynchronized), (\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\", TimeNotSynchronized), (\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\", TimeNotSynchronized), (\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\", TimeNotSynchronized)]","latency_us":462188,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:00.333611158Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"1.224005272s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4bd843ff-520c-4636-8acd-48f99945babf\", \"content-length\": \"124\", \"date\": \"Wed, 10 Jul 2024 23:54:59 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4bd843ff-520c-4636-8acd-48f99945babf\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 89504e37-0b08-442d-ace9-40ca97c7ca33, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5ae75945-01ab-454a-af38-a5dd1d09d270, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a390c63c-a68b-4301-b3c7-797628a3170a, kind: Service { id: 5ae75945-01ab-454a-af38-a5dd1d09d270 }, name: Name(\"ntp-5ae75945-01ab-454a-af38-a5dd1d09d270\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 108])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac507aba-8f76-47c6-bd22-6dff420215b2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2811064c-b3b2-438e-8e0c-aa9a3bfa436c, kind: Service { id: ac507aba-8f76-47c6-bd22-6dff420215b2 }, name: Name(\"external-dns-ac507aba-8f76-47c6-bd22-6dff420215b2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 139])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b39a7e8e-2dea-45da-9a7a-6977886585dc, kind: Service { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa }, name: Name(\"external-dns-b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 77])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: c108631e-f757-4a42-b510-28307153f426, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6ab9e3b5-f385-4e7e-9bc6-6c4f7a443782, kind: Service { id: c108631e-f757-4a42-b510-28307153f426 }, name: Name(\"nexus-c108631e-f757-4a42-b510-28307153f426\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 191])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6caaab71-9cdd-4faa-b082-987c9dd43604, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: ae27ee1d-0f9e-4fca-b104-9220b85a9848, kind: Service { id: 6caaab71-9cdd-4faa-b082-987c9dd43604 }, name: Name(\"nexus-6caaab71-9cdd-4faa-b082-987c9dd43604\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 147])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 37d6282c-2511-4c34-b419-e68b280bef44, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 97294434-1769-4a97-a97a-20d887e8c50a, kind: Service { id: 37d6282c-2511-4c34-b419-e68b280bef44 }, name: Name(\"nexus-37d6282c-2511-4c34-b419-e68b280bef44\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 88])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: abb1e589-7222-4a55-90ee-b0be940b410b, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: f864e092-4084-4307-8b83-2fd99a97943d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: fce1706b-b90a-4a5d-86b7-20320feb3342, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 32f00476-bb7b-4627-82ad-33c437d3a384, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: e736985c-90fa-417e-89f0-76ea8e60041f, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5baef1a-4fec-4af8-b323-da15f9e7b116, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 21592584-d863-4125-9acf-64a36ff982d8, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:01.559350756Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:01.55971033Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:01.55971862Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:01.583074344Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:01.583120284Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:01.601829218Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:01.601872708Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:01.618937044Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:01.618982774Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:01.6369606Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:01.63700183Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:01.653908334Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:01.653951634Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:01.672782029Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:01.67282652Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:01.69122056Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:01.6912625Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:01.711382659Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:01.711415059Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:01.729369975Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:01.729418515Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:01.74823264Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:01.7482731Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:01.767692171Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:01.767736092Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:01.786321074Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:01.786361174Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:01.805126508Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:01.805157789Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:01.823096994Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:01.823132894Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:01.841196841Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:01.841231681Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:01.859435409Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:01.85948216Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:01.878479956Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:01.878526567Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:01.897654055Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:01.897701345Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:01.915995974Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:01.916039395Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:01.934271433Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Reading ledger from /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:01.934323183Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:01.934498275Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:01.934615026Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:02.024905938Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:02.024951599Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:02.024958709Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:02.024966659Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:02.024972659Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:02.024979189Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:02.024997579Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:02.025002659Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:02.025010579Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:02.025017009Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:02.02502512Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:02.02503326Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:02.02503812Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:02.02504263Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:02.02505071Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:02.0250567Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:02.02506441Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:02.02507084Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:02.02507875Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:02.025196501Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"7414e59a-8644-4c3b-bb81-32cca508ed47","remote_addr":"[fd00:1122:3344:101::1]:63238","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\", TimeNotSynchronized), (\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\", TimeNotSynchronized), (\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\", TimeNotSynchronized), (\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\", TimeNotSynchronized), (\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\", TimeNotSynchronized), (\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\", TimeNotSynchronized), (\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\", TimeNotSynchronized), (\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\", TimeNotSynchronized), (\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\", TimeNotSynchronized), (\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\", TimeNotSynchronized), (\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\", TimeNotSynchronized), (\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\", TimeNotSynchronized), (\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\", TimeNotSynchronized), (\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\", TimeNotSynchronized), (\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\", TimeNotSynchronized), (\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\", TimeNotSynchronized), (\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\", TimeNotSynchronized), (\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\", TimeNotSynchronized), (\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\", TimeNotSynchronized)]","latency_us":465695,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:02.025387313Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"2.366137765s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7414e59a-8644-4c3b-bb81-32cca508ed47\", \"content-length\": \"124\", \"date\": \"Wed, 10 Jul 2024 23:55:01 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7414e59a-8644-4c3b-bb81-32cca508ed47\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 89504e37-0b08-442d-ace9-40ca97c7ca33, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5ae75945-01ab-454a-af38-a5dd1d09d270, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a390c63c-a68b-4301-b3c7-797628a3170a, kind: Service { id: 5ae75945-01ab-454a-af38-a5dd1d09d270 }, name: Name(\"ntp-5ae75945-01ab-454a-af38-a5dd1d09d270\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 108])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac507aba-8f76-47c6-bd22-6dff420215b2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2811064c-b3b2-438e-8e0c-aa9a3bfa436c, kind: Service { id: ac507aba-8f76-47c6-bd22-6dff420215b2 }, name: Name(\"external-dns-ac507aba-8f76-47c6-bd22-6dff420215b2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 139])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b39a7e8e-2dea-45da-9a7a-6977886585dc, kind: Service { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa }, name: Name(\"external-dns-b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 77])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: c108631e-f757-4a42-b510-28307153f426, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6ab9e3b5-f385-4e7e-9bc6-6c4f7a443782, kind: Service { id: c108631e-f757-4a42-b510-28307153f426 }, name: Name(\"nexus-c108631e-f757-4a42-b510-28307153f426\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 191])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6caaab71-9cdd-4faa-b082-987c9dd43604, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: ae27ee1d-0f9e-4fca-b104-9220b85a9848, kind: Service { id: 6caaab71-9cdd-4faa-b082-987c9dd43604 }, name: Name(\"nexus-6caaab71-9cdd-4faa-b082-987c9dd43604\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 147])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 37d6282c-2511-4c34-b419-e68b280bef44, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 97294434-1769-4a97-a97a-20d887e8c50a, kind: Service { id: 37d6282c-2511-4c34-b419-e68b280bef44 }, name: Name(\"nexus-37d6282c-2511-4c34-b419-e68b280bef44\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 88])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: abb1e589-7222-4a55-90ee-b0be940b410b, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: f864e092-4084-4307-8b83-2fd99a97943d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: fce1706b-b90a-4a5d-86b7-20320feb3342, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 32f00476-bb7b-4627-82ad-33c437d3a384, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: e736985c-90fa-417e-89f0-76ea8e60041f, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5baef1a-4fec-4af8-b323-da15f9e7b116, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 21592584-d863-4125-9acf-64a36ff982d8, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:04.393039768Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:04.393363552Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:04.393380822Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:04.414667243Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:04.414714064Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:04.433337256Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:04.433386477Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:04.451799867Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:04.451842097Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:04.469736952Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:04.469778822Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:04.487366954Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:04.487419324Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:04.50545671Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:04.505497281Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:04.522409225Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:04.522454875Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:04.539275938Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:04.539317859Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:04.557284404Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:04.557319804Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:04.574683983Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:04.574739294Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:04.592960452Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:04.593015042Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:04.610803116Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:04.610843046Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:04.627837131Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:04.627875462Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:04.645467341Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:04.645507879Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:04.663306337Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:04.663349887Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:04.680619995Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:04.680659325Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:04.698496179Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:04.69854069Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:04.717752599Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:04.717797489Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:04.734926886Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:04.734977506Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:04.754210845Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Reading ledger from /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:04.754264286Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:04.754458748Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:04.754571799Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:04.846379207Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:04.846432078Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:04.846439778Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:04.846446628Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:04.846452898Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:04.846461588Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:04.846469448Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:04.846475608Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:04.846483288Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:04.846491279Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:04.846503309Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:04.846508219Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:04.846513019Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:04.846520959Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:04.846526849Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:04.846532719Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:04.846537129Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:04.846541439Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:04.846548939Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:04.84658853Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"6578d138-7243-4c14-a549-cecba9d17e5e","remote_addr":"[fd00:1122:3344:101::1]:63238","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\", TimeNotSynchronized), (\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\", TimeNotSynchronized), (\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\", TimeNotSynchronized), (\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\", TimeNotSynchronized), (\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\", TimeNotSynchronized), (\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\", TimeNotSynchronized), (\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\", TimeNotSynchronized), (\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\", TimeNotSynchronized), (\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\", TimeNotSynchronized), (\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\", TimeNotSynchronized), (\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\", TimeNotSynchronized), (\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\", TimeNotSynchronized), (\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\", TimeNotSynchronized), (\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\", TimeNotSynchronized), (\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\", TimeNotSynchronized), (\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\", TimeNotSynchronized), (\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\", TimeNotSynchronized), (\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\", TimeNotSynchronized), (\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\", TimeNotSynchronized)]","latency_us":453401,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:04.846773302Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"2.481358759s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6578d138-7243-4c14-a549-cecba9d17e5e\", \"content-length\": \"124\", \"date\": \"Wed, 10 Jul 2024 23:55:04 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6578d138-7243-4c14-a549-cecba9d17e5e\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 89504e37-0b08-442d-ace9-40ca97c7ca33, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5ae75945-01ab-454a-af38-a5dd1d09d270, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a390c63c-a68b-4301-b3c7-797628a3170a, kind: Service { id: 5ae75945-01ab-454a-af38-a5dd1d09d270 }, name: Name(\"ntp-5ae75945-01ab-454a-af38-a5dd1d09d270\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 108])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac507aba-8f76-47c6-bd22-6dff420215b2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2811064c-b3b2-438e-8e0c-aa9a3bfa436c, kind: Service { id: ac507aba-8f76-47c6-bd22-6dff420215b2 }, name: Name(\"external-dns-ac507aba-8f76-47c6-bd22-6dff420215b2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 139])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b39a7e8e-2dea-45da-9a7a-6977886585dc, kind: Service { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa }, name: Name(\"external-dns-b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 77])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: c108631e-f757-4a42-b510-28307153f426, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6ab9e3b5-f385-4e7e-9bc6-6c4f7a443782, kind: Service { id: c108631e-f757-4a42-b510-28307153f426 }, name: Name(\"nexus-c108631e-f757-4a42-b510-28307153f426\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 191])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6caaab71-9cdd-4faa-b082-987c9dd43604, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: ae27ee1d-0f9e-4fca-b104-9220b85a9848, kind: Service { id: 6caaab71-9cdd-4faa-b082-987c9dd43604 }, name: Name(\"nexus-6caaab71-9cdd-4faa-b082-987c9dd43604\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 147])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 37d6282c-2511-4c34-b419-e68b280bef44, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 97294434-1769-4a97-a97a-20d887e8c50a, kind: Service { id: 37d6282c-2511-4c34-b419-e68b280bef44 }, name: Name(\"nexus-37d6282c-2511-4c34-b419-e68b280bef44\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 88])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: abb1e589-7222-4a55-90ee-b0be940b410b, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: f864e092-4084-4307-8b83-2fd99a97943d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: fce1706b-b90a-4a5d-86b7-20320feb3342, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 32f00476-bb7b-4627-82ad-33c437d3a384, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: e736985c-90fa-417e-89f0-76ea8e60041f, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5baef1a-4fec-4af8-b323-da15f9e7b116, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 21592584-d863-4125-9acf-64a36ff982d8, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:07.33510495Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:07.335467693Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:07.335475615Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:07.358029672Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:07.358074982Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:07.378834409Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:07.378877103Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:07.396861157Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:07.396904395Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:07.415793863Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:07.415843091Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:07.434061459Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:07.434105654Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:07.451458643Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:07.451502884Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:07.469421207Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:07.469466788Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:07.489040293Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:07.489079312Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:07.50703772Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:07.507081585Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:07.52569089Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:07.525732459Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:07.543717901Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:07.543768554Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:07.561488844Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:07.561523008Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:07.580093981Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:07.5801252Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:07.598060724Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:07.59810144Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:07.616489809Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:07.616521826Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:07.634443442Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:07.634479142Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:07.652437825Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:07.652478851Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:07.67072176Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:07.670758642Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:07.688594204Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:07.688638726Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:07.708220148Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Reading ledger from /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:07.708327724Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:07.708549005Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:07.708657325Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:07.802021705Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:07.802064877Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:07.802073212Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:07.802080881Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:07.802086524Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:07.802093931Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:07.802101262Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:07.80210716Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:07.802113122Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:07.802118841Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:07.802123135Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:07.802127335Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:07.8021319Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:07.802137938Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:07.802153032Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:07.802157503Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:07.802163616Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:07.802169522Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:07.802175353Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:07.802192173Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"987a625d-c03e-47d2-bf26-6c5a17734759","remote_addr":"[fd00:1122:3344:101::1]:63238","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\", TimeNotSynchronized), (\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\", TimeNotSynchronized), (\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\", TimeNotSynchronized), (\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\", TimeNotSynchronized), (\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\", TimeNotSynchronized), (\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\", TimeNotSynchronized), (\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\", TimeNotSynchronized), (\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\", TimeNotSynchronized), (\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\", TimeNotSynchronized), (\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\", TimeNotSynchronized), (\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\", TimeNotSynchronized), (\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\", TimeNotSynchronized), (\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\", TimeNotSynchronized), (\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\", TimeNotSynchronized), (\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\", TimeNotSynchronized), (\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\", TimeNotSynchronized), (\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\", TimeNotSynchronized), (\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\", TimeNotSynchronized), (\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\", TimeNotSynchronized)]","latency_us":465683,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:07.802382234Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"1.464563452s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"987a625d-c03e-47d2-bf26-6c5a17734759\", \"content-length\": \"124\", \"date\": \"Wed, 10 Jul 2024 23:55:07 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"987a625d-c03e-47d2-bf26-6c5a17734759\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 89504e37-0b08-442d-ace9-40ca97c7ca33, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5ae75945-01ab-454a-af38-a5dd1d09d270, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a390c63c-a68b-4301-b3c7-797628a3170a, kind: Service { id: 5ae75945-01ab-454a-af38-a5dd1d09d270 }, name: Name(\"ntp-5ae75945-01ab-454a-af38-a5dd1d09d270\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 108])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac507aba-8f76-47c6-bd22-6dff420215b2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2811064c-b3b2-438e-8e0c-aa9a3bfa436c, kind: Service { id: ac507aba-8f76-47c6-bd22-6dff420215b2 }, name: Name(\"external-dns-ac507aba-8f76-47c6-bd22-6dff420215b2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 139])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b39a7e8e-2dea-45da-9a7a-6977886585dc, kind: Service { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa }, name: Name(\"external-dns-b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 77])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: c108631e-f757-4a42-b510-28307153f426, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6ab9e3b5-f385-4e7e-9bc6-6c4f7a443782, kind: Service { id: c108631e-f757-4a42-b510-28307153f426 }, name: Name(\"nexus-c108631e-f757-4a42-b510-28307153f426\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 191])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6caaab71-9cdd-4faa-b082-987c9dd43604, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: ae27ee1d-0f9e-4fca-b104-9220b85a9848, kind: Service { id: 6caaab71-9cdd-4faa-b082-987c9dd43604 }, name: Name(\"nexus-6caaab71-9cdd-4faa-b082-987c9dd43604\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 147])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 37d6282c-2511-4c34-b419-e68b280bef44, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 97294434-1769-4a97-a97a-20d887e8c50a, kind: Service { id: 37d6282c-2511-4c34-b419-e68b280bef44 }, name: Name(\"nexus-37d6282c-2511-4c34-b419-e68b280bef44\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 88])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: abb1e589-7222-4a55-90ee-b0be940b410b, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: f864e092-4084-4307-8b83-2fd99a97943d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: fce1706b-b90a-4a5d-86b7-20320feb3342, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 32f00476-bb7b-4627-82ad-33c437d3a384, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: e736985c-90fa-417e-89f0-76ea8e60041f, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5baef1a-4fec-4af8-b323-da15f9e7b116, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 21592584-d863-4125-9acf-64a36ff982d8, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:09.271479273Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:09.271845189Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:09.271853495Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:09.293628556Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:09.293668793Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:09.312309664Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:09.312352743Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:09.330074022Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:09.330127911Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:09.347601368Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:09.347636693Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:09.366348279Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:09.366376347Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:09.38462972Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:09.384665281Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:09.402839704Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:09.402949849Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:09.421639443Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:09.4216764Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:09.440605052Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:09.440645665Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:09.459854946Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:09.459895906Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:09.478233441Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:09.478265363Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:09.49588746Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:09.495933257Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:09.513608918Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:09.513643118Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:09.532288199Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:09.532323196Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:09.550524985Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:09.550568194Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:09.568827405Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:09.568866893Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:09.586934662Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:09.586985607Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:09.605487103Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:09.605533069Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:09.623350534Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:09.623398113Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:09.643397189Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Reading ledger from /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:09.643451443Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:09.643687317Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:09.643794221Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:09.737180473Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:09.737232571Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:09.737237034Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:09.737262449Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:09.737268103Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:09.737273737Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:09.737280347Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:09.737285381Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:09.737291766Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:09.737296932Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:09.737310507Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:09.737314585Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:09.737318269Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:09.737323575Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:09.73733131Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:09.737337141Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:09.737341275Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:09.737345147Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:09.737355103Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:09.73738742Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"a165903f-6460-4776-8e06-0448b8e9c981","remote_addr":"[fd00:1122:3344:101::1]:63238","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\", TimeNotSynchronized), (\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\", TimeNotSynchronized), (\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\", TimeNotSynchronized), (\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\", TimeNotSynchronized), (\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\", TimeNotSynchronized), (\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\", TimeNotSynchronized), (\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\", TimeNotSynchronized), (\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\", TimeNotSynchronized), (\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\", TimeNotSynchronized), (\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\", TimeNotSynchronized), (\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\", TimeNotSynchronized), (\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\", TimeNotSynchronized), (\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\", TimeNotSynchronized), (\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\", TimeNotSynchronized), (\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\", TimeNotSynchronized), (\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\", TimeNotSynchronized), (\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\", TimeNotSynchronized), (\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\", TimeNotSynchronized), (\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\", TimeNotSynchronized)]","latency_us":464483,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:09.737553115Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"2.744356213s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a165903f-6460-4776-8e06-0448b8e9c981\", \"content-length\": \"124\", \"date\": \"Wed, 10 Jul 2024 23:55:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a165903f-6460-4776-8e06-0448b8e9c981\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 89504e37-0b08-442d-ace9-40ca97c7ca33, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5ae75945-01ab-454a-af38-a5dd1d09d270, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a390c63c-a68b-4301-b3c7-797628a3170a, kind: Service { id: 5ae75945-01ab-454a-af38-a5dd1d09d270 }, name: Name(\"ntp-5ae75945-01ab-454a-af38-a5dd1d09d270\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 108])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac507aba-8f76-47c6-bd22-6dff420215b2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2811064c-b3b2-438e-8e0c-aa9a3bfa436c, kind: Service { id: ac507aba-8f76-47c6-bd22-6dff420215b2 }, name: Name(\"external-dns-ac507aba-8f76-47c6-bd22-6dff420215b2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 139])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b39a7e8e-2dea-45da-9a7a-6977886585dc, kind: Service { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa }, name: Name(\"external-dns-b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 77])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: c108631e-f757-4a42-b510-28307153f426, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6ab9e3b5-f385-4e7e-9bc6-6c4f7a443782, kind: Service { id: c108631e-f757-4a42-b510-28307153f426 }, name: Name(\"nexus-c108631e-f757-4a42-b510-28307153f426\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 191])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6caaab71-9cdd-4faa-b082-987c9dd43604, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: ae27ee1d-0f9e-4fca-b104-9220b85a9848, kind: Service { id: 6caaab71-9cdd-4faa-b082-987c9dd43604 }, name: Name(\"nexus-6caaab71-9cdd-4faa-b082-987c9dd43604\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 147])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 37d6282c-2511-4c34-b419-e68b280bef44, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 97294434-1769-4a97-a97a-20d887e8c50a, kind: Service { id: 37d6282c-2511-4c34-b419-e68b280bef44 }, name: Name(\"nexus-37d6282c-2511-4c34-b419-e68b280bef44\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 88])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: abb1e589-7222-4a55-90ee-b0be940b410b, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: f864e092-4084-4307-8b83-2fd99a97943d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: fce1706b-b90a-4a5d-86b7-20320feb3342, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 32f00476-bb7b-4627-82ad-33c437d3a384, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: e736985c-90fa-417e-89f0-76ea8e60041f, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5baef1a-4fec-4af8-b323-da15f9e7b116, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 21592584-d863-4125-9acf-64a36ff982d8, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:12.490381299Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:12.490858041Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:12.490878105Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:12.513038235Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:12.513088092Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:12.533734252Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:12.533794065Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:12.55206763Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:12.552125155Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:12.569807191Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:12.569881806Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:12.587800808Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:12.587848639Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:12.60541021Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:12.60547128Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:12.624891325Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:12.624952919Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:12.64291536Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:12.642971077Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:12.6618547Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:12.661899972Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:12.680029948Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:12.680081117Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:12.698336409Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:12.698384531Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:12.716551206Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:12.716599001Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:12.734737173Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:12.734792158Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:12.752658772Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:12.752702131Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:12.771224174Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:12.771273927Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:12.789161781Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:12.789213748Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:12.807467098Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:12.807506989Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:12.826809108Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:12.826865255Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:12.844775095Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:12.844823929Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:12.863227896Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Reading ledger from /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:12.863349148Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:12.863558775Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:12.86368261Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:12.955952132Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:12.956018292Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:12.956032851Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:12.956047045Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:12.95605327Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:12.956059936Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:12.956082971Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:12.956088137Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:12.956094343Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:12.956099143Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:12.956103821Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:12.95610983Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:12.956117715Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:12.956124137Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:12.956128758Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:12.956135162Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:12.956146637Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:12.956157549Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:12.956161862Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:12.956172972Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"edb785a2-336b-4e5a-b519-3fee5db88558","remote_addr":"[fd00:1122:3344:101::1]:63238","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\", TimeNotSynchronized), (\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\", TimeNotSynchronized), (\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\", TimeNotSynchronized), (\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\", TimeNotSynchronized), (\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\", TimeNotSynchronized), (\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\", TimeNotSynchronized), (\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\", TimeNotSynchronized), (\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\", TimeNotSynchronized), (\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\", TimeNotSynchronized), (\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\", TimeNotSynchronized), (\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\", TimeNotSynchronized), (\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\", TimeNotSynchronized), (\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\", TimeNotSynchronized), (\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\", TimeNotSynchronized), (\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\", TimeNotSynchronized), (\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\", TimeNotSynchronized), (\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\", TimeNotSynchronized), (\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\", TimeNotSynchronized), (\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\", TimeNotSynchronized)]","latency_us":464398,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:12.956388843Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"4.106690242s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"edb785a2-336b-4e5a-b519-3fee5db88558\", \"content-length\": \"124\", \"date\": \"Wed, 10 Jul 2024 23:55:12 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"edb785a2-336b-4e5a-b519-3fee5db88558\" }"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:14.461825748Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:423","component":"producer-server","component":"MetricsManager","sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"oximeter/producer/src/lib.rs:423","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"2.672085987s"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 89504e37-0b08-442d-ace9-40ca97c7ca33, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5ae75945-01ab-454a-af38-a5dd1d09d270, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a390c63c-a68b-4301-b3c7-797628a3170a, kind: Service { id: 5ae75945-01ab-454a-af38-a5dd1d09d270 }, name: Name(\"ntp-5ae75945-01ab-454a-af38-a5dd1d09d270\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 108])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac507aba-8f76-47c6-bd22-6dff420215b2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2811064c-b3b2-438e-8e0c-aa9a3bfa436c, kind: Service { id: ac507aba-8f76-47c6-bd22-6dff420215b2 }, name: Name(\"external-dns-ac507aba-8f76-47c6-bd22-6dff420215b2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 139])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b39a7e8e-2dea-45da-9a7a-6977886585dc, kind: Service { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa }, name: Name(\"external-dns-b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 77])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: c108631e-f757-4a42-b510-28307153f426, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6ab9e3b5-f385-4e7e-9bc6-6c4f7a443782, kind: Service { id: c108631e-f757-4a42-b510-28307153f426 }, name: Name(\"nexus-c108631e-f757-4a42-b510-28307153f426\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 191])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6caaab71-9cdd-4faa-b082-987c9dd43604, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: ae27ee1d-0f9e-4fca-b104-9220b85a9848, kind: Service { id: 6caaab71-9cdd-4faa-b082-987c9dd43604 }, name: Name(\"nexus-6caaab71-9cdd-4faa-b082-987c9dd43604\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 147])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 37d6282c-2511-4c34-b419-e68b280bef44, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 97294434-1769-4a97-a97a-20d887e8c50a, kind: Service { id: 37d6282c-2511-4c34-b419-e68b280bef44 }, name: Name(\"nexus-37d6282c-2511-4c34-b419-e68b280bef44\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 88])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: abb1e589-7222-4a55-90ee-b0be940b410b, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: f864e092-4084-4307-8b83-2fd99a97943d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: fce1706b-b90a-4a5d-86b7-20320feb3342, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 32f00476-bb7b-4627-82ad-33c437d3a384, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: e736985c-90fa-417e-89f0-76ea8e60041f, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5baef1a-4fec-4af8-b323-da15f9e7b116, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 21592584-d863-4125-9acf-64a36ff982d8, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:17.074845302Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:17.075956339Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:17.075970748Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:17.096696583Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:17.096749684Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:17.115742411Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:17.115794828Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:17.133109687Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:17.13316921Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:17.151865283Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:17.151983214Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:17.171907967Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:17.172041262Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:17.190382925Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:17.190425779Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:17.208814623Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:17.208875467Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:17.227576665Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:17.227622875Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:17.248017487Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:17.248069107Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:17.26659706Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:17.26664432Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:17.284865648Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:17.28491572Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:17.302526323Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:17.302572767Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:17.319901121Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:17.319940788Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:17.337805423Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:17.337881586Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:17.35698822Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:17.357027652Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:17.37604753Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:17.376089258Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:17.39478185Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:17.394838831Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:17.413024431Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:17.413081545Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:17.431607166Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:17.431652803Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:17.450376073Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Reading ledger from /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:17.450422114Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:17.450602425Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:17.450786638Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:17.542252341Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:17.542324399Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:17.54233428Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:17.542343617Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:17.542358317Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:17.542363942Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:17.542371292Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:17.54237644Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:17.542405971Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:17.542415449Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:17.542422696Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:17.542428153Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:17.542433656Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:17.542442806Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:17.542448422Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:17.542455922Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:17.54246871Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:17.54247426Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:17.542481132Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:17.542492119Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"b5a7e3e6-2425-4034-8744-8d9211ed722a","remote_addr":"[fd00:1122:3344:101::1]:63238","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\", TimeNotSynchronized), (\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\", TimeNotSynchronized), (\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\", TimeNotSynchronized), (\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\", TimeNotSynchronized), (\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\", TimeNotSynchronized), (\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\", TimeNotSynchronized), (\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\", TimeNotSynchronized), (\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\", TimeNotSynchronized), (\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\", TimeNotSynchronized), (\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\", TimeNotSynchronized), (\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\", TimeNotSynchronized), (\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\", TimeNotSynchronized), (\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\", TimeNotSynchronized), (\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\", TimeNotSynchronized), (\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\", TimeNotSynchronized), (\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\", TimeNotSynchronized), (\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\", TimeNotSynchronized), (\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\", TimeNotSynchronized), (\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\", TimeNotSynchronized)]","latency_us":466173,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:17.54271309Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"2.475038084s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b5a7e3e6-2425-4034-8744-8d9211ed722a\", \"content-length\": \"124\", \"date\": \"Wed, 10 Jul 2024 23:55:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b5a7e3e6-2425-4034-8744-8d9211ed722a\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 89504e37-0b08-442d-ace9-40ca97c7ca33, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5ae75945-01ab-454a-af38-a5dd1d09d270, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a390c63c-a68b-4301-b3c7-797628a3170a, kind: Service { id: 5ae75945-01ab-454a-af38-a5dd1d09d270 }, name: Name(\"ntp-5ae75945-01ab-454a-af38-a5dd1d09d270\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 108])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac507aba-8f76-47c6-bd22-6dff420215b2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2811064c-b3b2-438e-8e0c-aa9a3bfa436c, kind: Service { id: ac507aba-8f76-47c6-bd22-6dff420215b2 }, name: Name(\"external-dns-ac507aba-8f76-47c6-bd22-6dff420215b2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 139])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b39a7e8e-2dea-45da-9a7a-6977886585dc, kind: Service { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa }, name: Name(\"external-dns-b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 77])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: c108631e-f757-4a42-b510-28307153f426, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6ab9e3b5-f385-4e7e-9bc6-6c4f7a443782, kind: Service { id: c108631e-f757-4a42-b510-28307153f426 }, name: Name(\"nexus-c108631e-f757-4a42-b510-28307153f426\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 191])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6caaab71-9cdd-4faa-b082-987c9dd43604, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: ae27ee1d-0f9e-4fca-b104-9220b85a9848, kind: Service { id: 6caaab71-9cdd-4faa-b082-987c9dd43604 }, name: Name(\"nexus-6caaab71-9cdd-4faa-b082-987c9dd43604\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 147])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 37d6282c-2511-4c34-b419-e68b280bef44, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 97294434-1769-4a97-a97a-20d887e8c50a, kind: Service { id: 37d6282c-2511-4c34-b419-e68b280bef44 }, name: Name(\"nexus-37d6282c-2511-4c34-b419-e68b280bef44\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 88])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: abb1e589-7222-4a55-90ee-b0be940b410b, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: f864e092-4084-4307-8b83-2fd99a97943d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: fce1706b-b90a-4a5d-86b7-20320feb3342, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 32f00476-bb7b-4627-82ad-33c437d3a384, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: e736985c-90fa-417e-89f0-76ea8e60041f, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5baef1a-4fec-4af8-b323-da15f9e7b116, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 21592584-d863-4125-9acf-64a36ff982d8, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:20.026304274Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:20.02681813Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:20.026833805Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:20.049172525Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:20.051655144Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:20.069792724Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:20.06990486Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:20.088623433Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:20.088676149Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:20.107605974Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:20.107655466Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:20.125089382Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:20.125135058Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:20.14333455Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:20.143383348Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:20.161312457Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:20.161361255Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:20.179422686Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:20.179465775Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:20.197370272Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:20.197408579Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:20.21556594Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:20.215613341Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:20.233721825Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:20.233760863Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:20.251779105Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:20.25183663Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:20.269895444Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:20.269955773Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:20.288140876Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:20.288182267Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:20.306255341Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:20.30630695Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:20.324766711Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:20.324814121Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:20.343416795Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:20.343458477Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:20.361740108Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:20.361776642Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:20.379426679Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:20.379467619Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:20.396857957Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Reading ledger from /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:20.396898936Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:20.397106912Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:20.397236364Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:20.491311059Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:20.49138771Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:20.491399851Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:20.491405476Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:20.491413135Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:20.491419135Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:20.491424245Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:20.491429354Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:20.491436235Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:20.491445546Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:20.49145418Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:20.49145973Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:20.49148343Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:20.491490939Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:20.49149618Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:20.491501711Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:20.49150713Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:20.49151643Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:20.491521643Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:20.49153294Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"b7b3b2cb-325f-45d4-8940-0e4de5853e79","remote_addr":"[fd00:1122:3344:101::1]:63238","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\", TimeNotSynchronized), (\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\", TimeNotSynchronized), (\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\", TimeNotSynchronized), (\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\", TimeNotSynchronized), (\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\", TimeNotSynchronized), (\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\", TimeNotSynchronized), (\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\", TimeNotSynchronized), (\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\", TimeNotSynchronized), (\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\", TimeNotSynchronized), (\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\", TimeNotSynchronized), (\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\", TimeNotSynchronized), (\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\", TimeNotSynchronized), (\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\", TimeNotSynchronized), (\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\", TimeNotSynchronized), (\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\", TimeNotSynchronized), (\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\", TimeNotSynchronized), (\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\", TimeNotSynchronized), (\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\", TimeNotSynchronized), (\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\", TimeNotSynchronized)]","latency_us":463798,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:20.491771123Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"6.820286937s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b7b3b2cb-325f-45d4-8940-0e4de5853e79\", \"content-length\": \"124\", \"date\": \"Wed, 10 Jul 2024 23:55:20 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b7b3b2cb-325f-45d4-8940-0e4de5853e79\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 89504e37-0b08-442d-ace9-40ca97c7ca33, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5ae75945-01ab-454a-af38-a5dd1d09d270, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a390c63c-a68b-4301-b3c7-797628a3170a, kind: Service { id: 5ae75945-01ab-454a-af38-a5dd1d09d270 }, name: Name(\"ntp-5ae75945-01ab-454a-af38-a5dd1d09d270\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 108])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac507aba-8f76-47c6-bd22-6dff420215b2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2811064c-b3b2-438e-8e0c-aa9a3bfa436c, kind: Service { id: ac507aba-8f76-47c6-bd22-6dff420215b2 }, name: Name(\"external-dns-ac507aba-8f76-47c6-bd22-6dff420215b2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 139])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b39a7e8e-2dea-45da-9a7a-6977886585dc, kind: Service { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa }, name: Name(\"external-dns-b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 77])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: c108631e-f757-4a42-b510-28307153f426, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6ab9e3b5-f385-4e7e-9bc6-6c4f7a443782, kind: Service { id: c108631e-f757-4a42-b510-28307153f426 }, name: Name(\"nexus-c108631e-f757-4a42-b510-28307153f426\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 191])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6caaab71-9cdd-4faa-b082-987c9dd43604, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: ae27ee1d-0f9e-4fca-b104-9220b85a9848, kind: Service { id: 6caaab71-9cdd-4faa-b082-987c9dd43604 }, name: Name(\"nexus-6caaab71-9cdd-4faa-b082-987c9dd43604\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 147])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 37d6282c-2511-4c34-b419-e68b280bef44, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 97294434-1769-4a97-a97a-20d887e8c50a, kind: Service { id: 37d6282c-2511-4c34-b419-e68b280bef44 }, name: Name(\"nexus-37d6282c-2511-4c34-b419-e68b280bef44\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 88])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: abb1e589-7222-4a55-90ee-b0be940b410b, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: f864e092-4084-4307-8b83-2fd99a97943d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: fce1706b-b90a-4a5d-86b7-20320feb3342, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 32f00476-bb7b-4627-82ad-33c437d3a384, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: e736985c-90fa-417e-89f0-76ea8e60041f, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5baef1a-4fec-4af8-b323-da15f9e7b116, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 21592584-d863-4125-9acf-64a36ff982d8, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:27.331785574Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:27.332631173Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:27.332644064Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:27.353589928Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:27.353650022Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:27.37447651Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:27.374527285Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:27.392950255Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:27.393004743Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:27.411425413Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:27.41147481Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:27.430843803Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:27.430892076Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:27.44875109Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:27.448863378Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:27.467625727Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:27.470021317Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:27.486143801Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:27.486190921Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:27.505174315Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:27.505227415Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:27.524112332Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:27.524175146Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:27.542414801Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:27.542468717Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:27.56086162Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:27.560910136Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:27.579460823Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:27.579508308Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:27.597466958Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:27.597517771Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:27.615400178Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:27.615453776Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:27.63415087Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:27.634202171Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:27.651921144Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:27.651967617Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:27.669852703Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:27.66990586Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:27.687633135Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:27.68767746Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:27.706138657Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Reading ledger from /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:27.706197719Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:27.70636859Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:27.706537944Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:27.79838892Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:27.798458371Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:27.798468346Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:27.798476859Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:27.798482634Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:27.798486524Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:27.798491793Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:27.798495646Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:27.798499377Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:27.798503146Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:27.798514293Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:27.798521324Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:27.798525328Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:27.798529012Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:27.798534835Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:27.798540478Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:27.798559219Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:27.798565031Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:27.798569006Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:27.79857286Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"16d95eac-6f5b-4f83-9095-ca7cbcbdb693","remote_addr":"[fd00:1122:3344:101::1]:63238","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\", TimeNotSynchronized), (\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\", TimeNotSynchronized), (\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\", TimeNotSynchronized), (\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\", TimeNotSynchronized), (\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\", TimeNotSynchronized), (\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\", TimeNotSynchronized), (\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\", TimeNotSynchronized), (\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\", TimeNotSynchronized), (\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\", TimeNotSynchronized), (\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\", TimeNotSynchronized), (\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\", TimeNotSynchronized), (\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\", TimeNotSynchronized), (\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\", TimeNotSynchronized), (\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\", TimeNotSynchronized), (\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\", TimeNotSynchronized), (\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\", TimeNotSynchronized), (\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\", TimeNotSynchronized), (\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\", TimeNotSynchronized), (\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\", TimeNotSynchronized)]","latency_us":465411,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:27.798809914Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"5.716632493s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"16d95eac-6f5b-4f83-9095-ca7cbcbdb693\", \"content-length\": \"124\", \"date\": \"Wed, 10 Jul 2024 23:55:27 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"16d95eac-6f5b-4f83-9095-ca7cbcbdb693\" }"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:28.156989925Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39944"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:28.157100158Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"7d52d9d3-3fae-4dd3-a998-17387e68fc7a","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39944","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":52,"response_code":"200"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:32.237936665Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:423","component":"producer-server","component":"MetricsManager","sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"oximeter/producer/src/lib.rs:423","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"4.190574082s"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 89504e37-0b08-442d-ace9-40ca97c7ca33, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5ae75945-01ab-454a-af38-a5dd1d09d270, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a390c63c-a68b-4301-b3c7-797628a3170a, kind: Service { id: 5ae75945-01ab-454a-af38-a5dd1d09d270 }, name: Name(\"ntp-5ae75945-01ab-454a-af38-a5dd1d09d270\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 108])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac507aba-8f76-47c6-bd22-6dff420215b2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2811064c-b3b2-438e-8e0c-aa9a3bfa436c, kind: Service { id: ac507aba-8f76-47c6-bd22-6dff420215b2 }, name: Name(\"external-dns-ac507aba-8f76-47c6-bd22-6dff420215b2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 139])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b39a7e8e-2dea-45da-9a7a-6977886585dc, kind: Service { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa }, name: Name(\"external-dns-b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 77])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: c108631e-f757-4a42-b510-28307153f426, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6ab9e3b5-f385-4e7e-9bc6-6c4f7a443782, kind: Service { id: c108631e-f757-4a42-b510-28307153f426 }, name: Name(\"nexus-c108631e-f757-4a42-b510-28307153f426\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 191])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6caaab71-9cdd-4faa-b082-987c9dd43604, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: ae27ee1d-0f9e-4fca-b104-9220b85a9848, kind: Service { id: 6caaab71-9cdd-4faa-b082-987c9dd43604 }, name: Name(\"nexus-6caaab71-9cdd-4faa-b082-987c9dd43604\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 147])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 37d6282c-2511-4c34-b419-e68b280bef44, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 97294434-1769-4a97-a97a-20d887e8c50a, kind: Service { id: 37d6282c-2511-4c34-b419-e68b280bef44 }, name: Name(\"nexus-37d6282c-2511-4c34-b419-e68b280bef44\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 88])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: abb1e589-7222-4a55-90ee-b0be940b410b, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: f864e092-4084-4307-8b83-2fd99a97943d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: fce1706b-b90a-4a5d-86b7-20320feb3342, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 32f00476-bb7b-4627-82ad-33c437d3a384, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: e736985c-90fa-417e-89f0-76ea8e60041f, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5baef1a-4fec-4af8-b323-da15f9e7b116, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 21592584-d863-4125-9acf-64a36ff982d8, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:33.532787477Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:33.533069489Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:33.533076877Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:33.554926073Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:33.554996563Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:33.576493142Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:33.576554353Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:33.599634365Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:33.599687447Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:33.619301969Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:33.619425376Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:33.639646035Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:33.63970448Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:33.659334618Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:33.659385956Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:33.679571382Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:33.679627586Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:33.700962965Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:33.701023819Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:33.722476336Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:33.722527646Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:33.743398957Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:33.743454054Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:33.764241311Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:33.764298002Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:33.785205647Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:33.785260285Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:33.805994387Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:33.806044207Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:33.826552257Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:33.826599076Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:33.848805252Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:33.848852877Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:33.870508646Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:33.870565047Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:33.891491299Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:33.891538972Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:33.910856526Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:33.910902979Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:33.929962545Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:33.930014989Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:33.950738178Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Reading ledger from /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:33.950797035Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:33.9509812Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:33.951098154Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:34.053294944Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:34.053363092Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:34.053371961Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:34.053377942Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:34.053385339Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:34.053392201Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:34.053398999Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:34.053411365Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:34.053417271Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:34.053422577Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:34.053429271Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:34.053434737Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:34.053441355Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:34.05344638Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:34.053466846Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:34.053471909Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:34.053479644Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:34.053485363Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:34.053491175Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:34.05361182Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"dcd754c2-5812-4e08-8c5f-03d839239bd9","remote_addr":"[fd00:1122:3344:101::1]:63238","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\", TimeNotSynchronized), (\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\", TimeNotSynchronized), (\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\", TimeNotSynchronized), (\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\", TimeNotSynchronized), (\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\", TimeNotSynchronized), (\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\", TimeNotSynchronized), (\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\", TimeNotSynchronized), (\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\", TimeNotSynchronized), (\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\", TimeNotSynchronized), (\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\", TimeNotSynchronized), (\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\", TimeNotSynchronized), (\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\", TimeNotSynchronized), (\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\", TimeNotSynchronized), (\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\", TimeNotSynchronized), (\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\", TimeNotSynchronized), (\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\", TimeNotSynchronized), (\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\", TimeNotSynchronized), (\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\", TimeNotSynchronized), (\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\", TimeNotSynchronized)]","latency_us":519267,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:34.053799097Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"4.915719521s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"dcd754c2-5812-4e08-8c5f-03d839239bd9\", \"content-length\": \"124\", \"date\": \"Wed, 10 Jul 2024 23:55:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"dcd754c2-5812-4e08-8c5f-03d839239bd9\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 89504e37-0b08-442d-ace9-40ca97c7ca33, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5ae75945-01ab-454a-af38-a5dd1d09d270, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a390c63c-a68b-4301-b3c7-797628a3170a, kind: Service { id: 5ae75945-01ab-454a-af38-a5dd1d09d270 }, name: Name(\"ntp-5ae75945-01ab-454a-af38-a5dd1d09d270\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 108])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac507aba-8f76-47c6-bd22-6dff420215b2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2811064c-b3b2-438e-8e0c-aa9a3bfa436c, kind: Service { id: ac507aba-8f76-47c6-bd22-6dff420215b2 }, name: Name(\"external-dns-ac507aba-8f76-47c6-bd22-6dff420215b2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 139])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b39a7e8e-2dea-45da-9a7a-6977886585dc, kind: Service { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa }, name: Name(\"external-dns-b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 77])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: c108631e-f757-4a42-b510-28307153f426, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6ab9e3b5-f385-4e7e-9bc6-6c4f7a443782, kind: Service { id: c108631e-f757-4a42-b510-28307153f426 }, name: Name(\"nexus-c108631e-f757-4a42-b510-28307153f426\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 191])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6caaab71-9cdd-4faa-b082-987c9dd43604, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: ae27ee1d-0f9e-4fca-b104-9220b85a9848, kind: Service { id: 6caaab71-9cdd-4faa-b082-987c9dd43604 }, name: Name(\"nexus-6caaab71-9cdd-4faa-b082-987c9dd43604\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 147])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 37d6282c-2511-4c34-b419-e68b280bef44, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 97294434-1769-4a97-a97a-20d887e8c50a, kind: Service { id: 37d6282c-2511-4c34-b419-e68b280bef44 }, name: Name(\"nexus-37d6282c-2511-4c34-b419-e68b280bef44\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 88])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: abb1e589-7222-4a55-90ee-b0be940b410b, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: f864e092-4084-4307-8b83-2fd99a97943d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: fce1706b-b90a-4a5d-86b7-20320feb3342, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 32f00476-bb7b-4627-82ad-33c437d3a384, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: e736985c-90fa-417e-89f0-76ea8e60041f, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5baef1a-4fec-4af8-b323-da15f9e7b116, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 21592584-d863-4125-9acf-64a36ff982d8, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:38.984129497Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:38.984990291Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:38.984998569Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:39.006312151Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:39.006360226Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:39.027015898Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:39.027062445Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:39.04631696Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:39.046367089Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:39.064222991Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:39.06426471Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:39.082692156Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:39.082739472Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:39.101003865Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:39.103272209Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:39.118771264Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:39.118814905Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:39.137999135Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:39.138051439Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:39.157277911Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:39.157322611Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:39.176679315Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:39.176721653Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:39.194932915Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:39.19497815Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:39.212951466Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:39.213004537Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:39.231401835Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:39.23380251Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:39.250688372Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:39.250735397Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:39.268763875Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:39.268811388Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:39.28664922Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:39.288858908Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:39.304561156Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:39.304679742Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:39.322349249Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:39.324587328Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:39.340826596Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:39.340865531Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:39.358427277Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Reading ledger from /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:39.358487428Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:39.358680723Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:39.358853936Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:39.446770504Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:39.44686284Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:39.446875121Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:39.44690319Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:39.446912902Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:39.446929459Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:39.446937971Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:39.446949372Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:39.446961905Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:39.446973231Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:39.447052355Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:39.447071264Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:39.44707838Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:39.447087136Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:39.447097786Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:39.447108034Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:39.44711483Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:39.447125471Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:39.44715204Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:39.447159231Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"1f08e598-495e-4f94-827b-01dcf5f65691","remote_addr":"[fd00:1122:3344:101::1]:63238","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\", TimeNotSynchronized), (\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\", TimeNotSynchronized), (\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\", TimeNotSynchronized), (\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\", TimeNotSynchronized), (\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\", TimeNotSynchronized), (\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\", TimeNotSynchronized), (\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\", TimeNotSynchronized), (\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\", TimeNotSynchronized), (\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\", TimeNotSynchronized), (\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\", TimeNotSynchronized), (\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\", TimeNotSynchronized), (\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\", TimeNotSynchronized), (\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\", TimeNotSynchronized), (\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\", TimeNotSynchronized), (\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\", TimeNotSynchronized), (\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\", TimeNotSynchronized), (\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\", TimeNotSynchronized), (\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\", TimeNotSynchronized), (\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\", TimeNotSynchronized)]","latency_us":461661,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:39.447356604Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"13.346418859s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1f08e598-495e-4f94-827b-01dcf5f65691\", \"content-length\": \"124\", \"date\": \"Wed, 10 Jul 2024 23:55:38 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1f08e598-495e-4f94-827b-01dcf5f65691\" }"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:51.517498163Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:423","component":"producer-server","component":"MetricsManager","sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"oximeter/producer/src/lib.rs:423","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"11.086180189s"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 89504e37-0b08-442d-ace9-40ca97c7ca33, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5ae75945-01ab-454a-af38-a5dd1d09d270, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a390c63c-a68b-4301-b3c7-797628a3170a, kind: Service { id: 5ae75945-01ab-454a-af38-a5dd1d09d270 }, name: Name(\"ntp-5ae75945-01ab-454a-af38-a5dd1d09d270\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 108])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac507aba-8f76-47c6-bd22-6dff420215b2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2811064c-b3b2-438e-8e0c-aa9a3bfa436c, kind: Service { id: ac507aba-8f76-47c6-bd22-6dff420215b2 }, name: Name(\"external-dns-ac507aba-8f76-47c6-bd22-6dff420215b2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 139])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b39a7e8e-2dea-45da-9a7a-6977886585dc, kind: Service { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa }, name: Name(\"external-dns-b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 77])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: c108631e-f757-4a42-b510-28307153f426, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6ab9e3b5-f385-4e7e-9bc6-6c4f7a443782, kind: Service { id: c108631e-f757-4a42-b510-28307153f426 }, name: Name(\"nexus-c108631e-f757-4a42-b510-28307153f426\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 191])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6caaab71-9cdd-4faa-b082-987c9dd43604, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: ae27ee1d-0f9e-4fca-b104-9220b85a9848, kind: Service { id: 6caaab71-9cdd-4faa-b082-987c9dd43604 }, name: Name(\"nexus-6caaab71-9cdd-4faa-b082-987c9dd43604\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 147])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 37d6282c-2511-4c34-b419-e68b280bef44, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 97294434-1769-4a97-a97a-20d887e8c50a, kind: Service { id: 37d6282c-2511-4c34-b419-e68b280bef44 }, name: Name(\"nexus-37d6282c-2511-4c34-b419-e68b280bef44\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 88])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: abb1e589-7222-4a55-90ee-b0be940b410b, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: f864e092-4084-4307-8b83-2fd99a97943d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: fce1706b-b90a-4a5d-86b7-20320feb3342, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 32f00476-bb7b-4627-82ad-33c437d3a384, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: e736985c-90fa-417e-89f0-76ea8e60041f, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5baef1a-4fec-4af8-b323-da15f9e7b116, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 21592584-d863-4125-9acf-64a36ff982d8, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:52.831564084Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:52.831834958Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:52.83184498Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:52.853875471Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:52.853926968Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:52.873854508Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:52.873906259Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:52.891452625Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:52.891491297Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:52.908525848Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:52.908568927Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:52.927375612Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:52.927427766Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:52.944690818Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:52.944739043Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:52.962523757Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:52.962567978Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:52.979725993Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:52.979779084Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:52.99844219Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:52.998482747Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:53.017338442Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:53.017374648Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:53.035582979Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:53.035616523Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:53.054477094Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:53.054518832Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:53.072828971Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:53.072873953Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:53.091320474Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:53.091380784Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:53.110087716Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:53.110130158Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:53.128060823Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:53.128132214Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:53.145646282Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:53.145685611Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:53.163393177Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:53.163430875Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:53.181464447Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:53.181500476Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:53.200005116Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Reading ledger from /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:53.200048176Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:53.20029491Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:53.200418746Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:53.291364589Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:53.291426165Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:53.291437518Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:53.29144439Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:53.291472618Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:53.291481993Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:53.291488452Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:53.29149674Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:53.291503125Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:53.291518144Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:53.291526394Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:53.291538131Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:53.291544375Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:53.291548763Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:53.291554959Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:53.291561213Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:53.291569453Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:53.291577394Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:53.291582091Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:53.291630007Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"53da9c30-3f2c-4697-94b4-3f76bb696f73","remote_addr":"[fd00:1122:3344:101::1]:63238","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\", TimeNotSynchronized), (\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\", TimeNotSynchronized), (\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\", TimeNotSynchronized), (\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\", TimeNotSynchronized), (\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\", TimeNotSynchronized), (\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\", TimeNotSynchronized), (\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\", TimeNotSynchronized), (\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\", TimeNotSynchronized), (\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\", TimeNotSynchronized), (\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\", TimeNotSynchronized), (\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\", TimeNotSynchronized), (\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\", TimeNotSynchronized), (\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\", TimeNotSynchronized), (\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\", TimeNotSynchronized), (\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\", TimeNotSynchronized), (\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\", TimeNotSynchronized), (\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\", TimeNotSynchronized), (\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\", TimeNotSynchronized), (\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\", TimeNotSynchronized)]","latency_us":458707,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:55:53.291813919Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"14.035418129s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"53da9c30-3f2c-4697-94b4-3f76bb696f73\", \"content-length\": \"124\", \"date\": \"Wed, 10 Jul 2024 23:55:52 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"53da9c30-3f2c-4697-94b4-3f76bb696f73\" }"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:58.270669422Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:40490"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:55:58.270732751Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"d8efe4c0-6f12-46f5-880c-bde81660f99c","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:40490","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":46,"response_code":"200"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 89504e37-0b08-442d-ace9-40ca97c7ca33, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5ae75945-01ab-454a-af38-a5dd1d09d270, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a390c63c-a68b-4301-b3c7-797628a3170a, kind: Service { id: 5ae75945-01ab-454a-af38-a5dd1d09d270 }, name: Name(\"ntp-5ae75945-01ab-454a-af38-a5dd1d09d270\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 108])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac507aba-8f76-47c6-bd22-6dff420215b2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2811064c-b3b2-438e-8e0c-aa9a3bfa436c, kind: Service { id: ac507aba-8f76-47c6-bd22-6dff420215b2 }, name: Name(\"external-dns-ac507aba-8f76-47c6-bd22-6dff420215b2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 139])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b39a7e8e-2dea-45da-9a7a-6977886585dc, kind: Service { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa }, name: Name(\"external-dns-b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 77])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: c108631e-f757-4a42-b510-28307153f426, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6ab9e3b5-f385-4e7e-9bc6-6c4f7a443782, kind: Service { id: c108631e-f757-4a42-b510-28307153f426 }, name: Name(\"nexus-c108631e-f757-4a42-b510-28307153f426\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 191])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6caaab71-9cdd-4faa-b082-987c9dd43604, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: ae27ee1d-0f9e-4fca-b104-9220b85a9848, kind: Service { id: 6caaab71-9cdd-4faa-b082-987c9dd43604 }, name: Name(\"nexus-6caaab71-9cdd-4faa-b082-987c9dd43604\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 147])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 37d6282c-2511-4c34-b419-e68b280bef44, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 97294434-1769-4a97-a97a-20d887e8c50a, kind: Service { id: 37d6282c-2511-4c34-b419-e68b280bef44 }, name: Name(\"nexus-37d6282c-2511-4c34-b419-e68b280bef44\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 88])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: abb1e589-7222-4a55-90ee-b0be940b410b, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: f864e092-4084-4307-8b83-2fd99a97943d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: fce1706b-b90a-4a5d-86b7-20320feb3342, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 32f00476-bb7b-4627-82ad-33c437d3a384, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: e736985c-90fa-417e-89f0-76ea8e60041f, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5baef1a-4fec-4af8-b323-da15f9e7b116, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 21592584-d863-4125-9acf-64a36ff982d8, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:07.366288094Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:07.367311618Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:07.367319203Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:07.390100698Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:07.390151745Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:07.410167584Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:07.410227745Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:07.428295101Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:07.428344563Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:07.446819354Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:07.446870936Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:07.46519354Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:07.465318796Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:07.483246891Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:07.483359771Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:07.502028725Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:07.502084853Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:07.520787647Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:07.520837363Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:07.541903879Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:07.54195998Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:07.560682276Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:07.560741198Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:07.580062046Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:07.580114547Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:07.59866415Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:07.598714326Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:07.616618468Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:07.616758151Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:07.635382468Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:07.637686012Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:07.655401886Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:07.655456102Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:07.675120804Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:07.675172554Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:07.693493469Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:07.693546588Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:07.712244885Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:07.712292135Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:07.730659873Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:07.730703111Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:07.748798851Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Reading ledger from /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:07.748874677Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:07.749201038Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:07.749337943Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:07.839525475Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:07.839603466Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:07.839619479Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:07.839627607Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:07.839636495Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:07.839670338Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:07.83967947Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:07.839695446Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:07.83970288Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:07.839708317Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:07.839720524Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:07.839728099Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:07.839737736Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:07.839745171Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:07.839750683Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:07.839758042Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:07.839765271Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:07.839772687Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:07.839777825Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:07.839796396Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"1af3054c-5559-4aee-bbd3-f5283ce67b54","remote_addr":"[fd00:1122:3344:101::1]:63238","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\", TimeNotSynchronized), (\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\", TimeNotSynchronized), (\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\", TimeNotSynchronized), (\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\", TimeNotSynchronized), (\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\", TimeNotSynchronized), (\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\", TimeNotSynchronized), (\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\", TimeNotSynchronized), (\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\", TimeNotSynchronized), (\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\", TimeNotSynchronized), (\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\", TimeNotSynchronized), (\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\", TimeNotSynchronized), (\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\", TimeNotSynchronized), (\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\", TimeNotSynchronized), (\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\", TimeNotSynchronized), (\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\", TimeNotSynchronized), (\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\", TimeNotSynchronized), (\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\", TimeNotSynchronized), (\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\", TimeNotSynchronized), (\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\", TimeNotSynchronized)]","latency_us":472111,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:07.839975189Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"10.698374386s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1af3054c-5559-4aee-bbd3-f5283ce67b54\", \"content-length\": \"124\", \"date\": \"Wed, 10 Jul 2024 23:56:07 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1af3054c-5559-4aee-bbd3-f5283ce67b54\" }"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:17.709301978Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:423","component":"producer-server","component":"MetricsManager","sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"oximeter/producer/src/lib.rs:423","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"12.72199516s"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 89504e37-0b08-442d-ace9-40ca97c7ca33, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5ae75945-01ab-454a-af38-a5dd1d09d270, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a390c63c-a68b-4301-b3c7-797628a3170a, kind: Service { id: 5ae75945-01ab-454a-af38-a5dd1d09d270 }, name: Name(\"ntp-5ae75945-01ab-454a-af38-a5dd1d09d270\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 108])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac507aba-8f76-47c6-bd22-6dff420215b2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2811064c-b3b2-438e-8e0c-aa9a3bfa436c, kind: Service { id: ac507aba-8f76-47c6-bd22-6dff420215b2 }, name: Name(\"external-dns-ac507aba-8f76-47c6-bd22-6dff420215b2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 139])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b39a7e8e-2dea-45da-9a7a-6977886585dc, kind: Service { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa }, name: Name(\"external-dns-b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 77])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: c108631e-f757-4a42-b510-28307153f426, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6ab9e3b5-f385-4e7e-9bc6-6c4f7a443782, kind: Service { id: c108631e-f757-4a42-b510-28307153f426 }, name: Name(\"nexus-c108631e-f757-4a42-b510-28307153f426\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 191])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6caaab71-9cdd-4faa-b082-987c9dd43604, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: ae27ee1d-0f9e-4fca-b104-9220b85a9848, kind: Service { id: 6caaab71-9cdd-4faa-b082-987c9dd43604 }, name: Name(\"nexus-6caaab71-9cdd-4faa-b082-987c9dd43604\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 147])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 37d6282c-2511-4c34-b419-e68b280bef44, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 97294434-1769-4a97-a97a-20d887e8c50a, kind: Service { id: 37d6282c-2511-4c34-b419-e68b280bef44 }, name: Name(\"nexus-37d6282c-2511-4c34-b419-e68b280bef44\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 88])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: abb1e589-7222-4a55-90ee-b0be940b410b, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: f864e092-4084-4307-8b83-2fd99a97943d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: fce1706b-b90a-4a5d-86b7-20320feb3342, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 32f00476-bb7b-4627-82ad-33c437d3a384, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: e736985c-90fa-417e-89f0-76ea8e60041f, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5baef1a-4fec-4af8-b323-da15f9e7b116, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 21592584-d863-4125-9acf-64a36ff982d8, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:18.56817829Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:18.568481555Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:18.568489899Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:18.590573699Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:18.590652262Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:18.611471177Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:18.611520321Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:18.630945099Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:18.630990418Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:18.649668659Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:18.649745057Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:18.667660496Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:18.6677149Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:18.687540457Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:18.687585393Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:18.706896278Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:18.70695252Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:18.724589731Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:18.724641575Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:18.74422796Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:18.744274873Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:18.762164389Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:18.762220179Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:18.779967647Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:18.780018Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:18.7984786Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:18.798530322Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:18.817241856Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:18.81729252Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:18.835861895Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:18.835908396Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:18.85361551Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:18.853659582Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:18.87194797Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:18.872001905Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:18.891105005Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:18.891156071Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:18.90942063Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:18.909489911Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:18.927549453Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:18.92760741Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:18.945961913Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Reading ledger from /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:18.946026254Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:18.94633712Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:18.946488612Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:19.041629675Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:19.041698029Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:19.041708061Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:19.041721054Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:19.041729408Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:19.041736017Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:19.041742092Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:19.041762343Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:19.041770227Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:19.041777174Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:19.041781852Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:19.041788058Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:19.041792549Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:19.041797312Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:19.041803462Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:19.041811262Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:19.041817449Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:19.041822222Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:19.041826722Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:19.041875059Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"217bb49d-0273-4112-ab61-b613c1ccd7eb","remote_addr":"[fd00:1122:3344:101::1]:63238","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\", TimeNotSynchronized), (\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\", TimeNotSynchronized), (\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\", TimeNotSynchronized), (\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\", TimeNotSynchronized), (\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\", TimeNotSynchronized), (\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\", TimeNotSynchronized), (\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\", TimeNotSynchronized), (\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\", TimeNotSynchronized), (\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\", TimeNotSynchronized), (\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\", TimeNotSynchronized), (\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\", TimeNotSynchronized), (\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\", TimeNotSynchronized), (\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\", TimeNotSynchronized), (\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\", TimeNotSynchronized), (\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\", TimeNotSynchronized), (\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\", TimeNotSynchronized), (\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\", TimeNotSynchronized), (\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\", TimeNotSynchronized), (\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\", TimeNotSynchronized)]","latency_us":472327,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:19.042081958Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"15.241189846s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"217bb49d-0273-4112-ab61-b613c1ccd7eb\", \"content-length\": \"124\", \"date\": \"Wed, 10 Jul 2024 23:56:19 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"217bb49d-0273-4112-ab61-b613c1ccd7eb\" }"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:28.382322246Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36877"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:28.382469045Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"db65cb76-7411-4716-8a94-0a3229906efb","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36877","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":49,"response_code":"200"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 89504e37-0b08-442d-ace9-40ca97c7ca33, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5ae75945-01ab-454a-af38-a5dd1d09d270, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a390c63c-a68b-4301-b3c7-797628a3170a, kind: Service { id: 5ae75945-01ab-454a-af38-a5dd1d09d270 }, name: Name(\"ntp-5ae75945-01ab-454a-af38-a5dd1d09d270\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 108])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac507aba-8f76-47c6-bd22-6dff420215b2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2811064c-b3b2-438e-8e0c-aa9a3bfa436c, kind: Service { id: ac507aba-8f76-47c6-bd22-6dff420215b2 }, name: Name(\"external-dns-ac507aba-8f76-47c6-bd22-6dff420215b2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 139])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b39a7e8e-2dea-45da-9a7a-6977886585dc, kind: Service { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa }, name: Name(\"external-dns-b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 77])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: c108631e-f757-4a42-b510-28307153f426, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6ab9e3b5-f385-4e7e-9bc6-6c4f7a443782, kind: Service { id: c108631e-f757-4a42-b510-28307153f426 }, name: Name(\"nexus-c108631e-f757-4a42-b510-28307153f426\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 191])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6caaab71-9cdd-4faa-b082-987c9dd43604, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: ae27ee1d-0f9e-4fca-b104-9220b85a9848, kind: Service { id: 6caaab71-9cdd-4faa-b082-987c9dd43604 }, name: Name(\"nexus-6caaab71-9cdd-4faa-b082-987c9dd43604\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 147])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 37d6282c-2511-4c34-b419-e68b280bef44, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 97294434-1769-4a97-a97a-20d887e8c50a, kind: Service { id: 37d6282c-2511-4c34-b419-e68b280bef44 }, name: Name(\"nexus-37d6282c-2511-4c34-b419-e68b280bef44\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 88])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: abb1e589-7222-4a55-90ee-b0be940b410b, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: f864e092-4084-4307-8b83-2fd99a97943d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: fce1706b-b90a-4a5d-86b7-20320feb3342, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 32f00476-bb7b-4627-82ad-33c437d3a384, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: e736985c-90fa-417e-89f0-76ea8e60041f, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5baef1a-4fec-4af8-b323-da15f9e7b116, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 21592584-d863-4125-9acf-64a36ff982d8, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:34.32535006Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:34.325695456Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:34.325703341Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:34.346251827Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:34.346300906Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:34.36527307Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:34.36532722Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:34.382843996Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:34.38501595Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:34.40024756Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:34.400299282Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:34.418001948Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:34.419872258Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:34.436819862Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:34.436947176Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:34.454270053Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:34.45431872Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:34.471809659Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:34.471856479Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:34.489516893Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:34.489573002Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:34.507325464Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:34.507369339Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:34.525478552Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:34.527707319Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:34.543000103Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:34.543057554Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:34.561017491Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:34.561072973Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:34.578867033Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:34.578916158Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:34.596993057Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:34.597042069Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:34.614233833Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:34.614291761Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:34.631634926Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:34.631705952Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:34.6492643Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:34.64930991Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:34.667917841Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:34.667971636Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:34.686503174Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Reading ledger from /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:34.686626212Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:34.686910577Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:34.687061207Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:34.777976087Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:34.778041047Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:34.778048632Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:34.778054538Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:34.778061485Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:34.77806831Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:34.77807522Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:34.778087154Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:34.778099023Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:34.778104657Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:34.778109982Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:34.778119245Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:34.778123079Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:34.778126904Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:34.778131621Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:34.778136571Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:34.778145899Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:34.778149827Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:34.778153596Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:34.778158977Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"d86685a1-e4a7-43ee-8f00-a1b63a897928","remote_addr":"[fd00:1122:3344:101::1]:63238","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\", TimeNotSynchronized), (\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\", TimeNotSynchronized), (\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\", TimeNotSynchronized), (\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\", TimeNotSynchronized), (\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\", TimeNotSynchronized), (\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\", TimeNotSynchronized), (\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\", TimeNotSynchronized), (\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\", TimeNotSynchronized), (\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\", TimeNotSynchronized), (\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\", TimeNotSynchronized), (\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\", TimeNotSynchronized), (\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\", TimeNotSynchronized), (\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\", TimeNotSynchronized), (\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\", TimeNotSynchronized), (\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\", TimeNotSynchronized), (\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\", TimeNotSynchronized), (\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\", TimeNotSynchronized), (\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\", TimeNotSynchronized), (\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\", TimeNotSynchronized)]","latency_us":451433,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:34.778320463Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"27.188493898s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d86685a1-e4a7-43ee-8f00-a1b63a897928\", \"content-length\": \"124\", \"date\": \"Wed, 10 Jul 2024 23:56:34 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d86685a1-e4a7-43ee-8f00-a1b63a897928\" }"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:56:45.540949655Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:423","component":"producer-server","component":"MetricsManager","sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"oximeter/producer/src/lib.rs:423","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"28.865563495s"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:58.49440115Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:47528"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:56:58.494489286Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"f52005ca-9afa-4e77-8903-f053b032c3d8","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:47528","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":61,"response_code":"200"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 89504e37-0b08-442d-ace9-40ca97c7ca33, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5ae75945-01ab-454a-af38-a5dd1d09d270, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a390c63c-a68b-4301-b3c7-797628a3170a, kind: Service { id: 5ae75945-01ab-454a-af38-a5dd1d09d270 }, name: Name(\"ntp-5ae75945-01ab-454a-af38-a5dd1d09d270\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 108])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac507aba-8f76-47c6-bd22-6dff420215b2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2811064c-b3b2-438e-8e0c-aa9a3bfa436c, kind: Service { id: ac507aba-8f76-47c6-bd22-6dff420215b2 }, name: Name(\"external-dns-ac507aba-8f76-47c6-bd22-6dff420215b2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 139])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b39a7e8e-2dea-45da-9a7a-6977886585dc, kind: Service { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa }, name: Name(\"external-dns-b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 77])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: c108631e-f757-4a42-b510-28307153f426, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6ab9e3b5-f385-4e7e-9bc6-6c4f7a443782, kind: Service { id: c108631e-f757-4a42-b510-28307153f426 }, name: Name(\"nexus-c108631e-f757-4a42-b510-28307153f426\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 191])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6caaab71-9cdd-4faa-b082-987c9dd43604, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: ae27ee1d-0f9e-4fca-b104-9220b85a9848, kind: Service { id: 6caaab71-9cdd-4faa-b082-987c9dd43604 }, name: Name(\"nexus-6caaab71-9cdd-4faa-b082-987c9dd43604\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 147])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 37d6282c-2511-4c34-b419-e68b280bef44, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 97294434-1769-4a97-a97a-20d887e8c50a, kind: Service { id: 37d6282c-2511-4c34-b419-e68b280bef44 }, name: Name(\"nexus-37d6282c-2511-4c34-b419-e68b280bef44\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 88])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: abb1e589-7222-4a55-90ee-b0be940b410b, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: f864e092-4084-4307-8b83-2fd99a97943d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: fce1706b-b90a-4a5d-86b7-20320feb3342, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 32f00476-bb7b-4627-82ad-33c437d3a384, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: e736985c-90fa-417e-89f0-76ea8e60041f, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5baef1a-4fec-4af8-b323-da15f9e7b116, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 21592584-d863-4125-9acf-64a36ff982d8, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:02.042320384Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:02.042765907Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:02.042773117Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:02.066759979Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:02.066826486Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:02.089197383Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:02.089250577Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:02.10933099Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:02.109395893Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:02.128890918Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:02.128962103Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:02.14742708Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:02.147475286Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:02.167220365Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:02.167335506Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:02.186423612Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:02.186480199Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:02.205132234Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:02.20519486Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:02.224225071Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:02.224285325Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:02.243328594Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:02.245754581Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:02.262262561Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:02.262308217Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:02.281715401Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:02.281767658Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:02.299949335Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:02.300005295Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:02.318397955Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:02.31844004Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:02.33871089Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:02.338842325Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:02.358291481Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:02.358344516Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:02.377366365Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:02.377417778Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:02.395663299Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:02.398103001Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:02.41486394Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:02.414922224Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:02.434493031Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Reading ledger from /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:02.434569991Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:02.434852238Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:02.435051937Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:02.528529641Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:02.528613661Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:02.528623805Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:02.528646736Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:02.528656176Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:02.528664023Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:02.528688351Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:02.528703981Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:02.528713459Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:02.528722834Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:02.528732546Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:02.528741846Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:02.528748634Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:02.528756462Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:02.528769081Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:02.528776675Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:02.528785816Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:02.52880331Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:02.5288123Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:02.5288381Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"154272f7-18fc-4c39-9a8d-c098f71df2a6","remote_addr":"[fd00:1122:3344:101::1]:63238","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\", TimeNotSynchronized), (\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\", TimeNotSynchronized), (\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\", TimeNotSynchronized), (\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\", TimeNotSynchronized), (\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\", TimeNotSynchronized), (\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\", TimeNotSynchronized), (\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\", TimeNotSynchronized), (\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\", TimeNotSynchronized), (\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\", TimeNotSynchronized), (\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\", TimeNotSynchronized), (\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\", TimeNotSynchronized), (\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\", TimeNotSynchronized), (\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\", TimeNotSynchronized), (\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\", TimeNotSynchronized), (\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\", TimeNotSynchronized), (\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\", TimeNotSynchronized), (\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\", TimeNotSynchronized), (\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\", TimeNotSynchronized), (\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\", TimeNotSynchronized)]","latency_us":485027,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:02.529027674Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"25.316891794s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"154272f7-18fc-4c39-9a8d-c098f71df2a6\", \"content-length\": \"124\", \"date\": \"Wed, 10 Jul 2024 23:57:02 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"154272f7-18fc-4c39-9a8d-c098f71df2a6\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 89504e37-0b08-442d-ace9-40ca97c7ca33, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5ae75945-01ab-454a-af38-a5dd1d09d270, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a390c63c-a68b-4301-b3c7-797628a3170a, kind: Service { id: 5ae75945-01ab-454a-af38-a5dd1d09d270 }, name: Name(\"ntp-5ae75945-01ab-454a-af38-a5dd1d09d270\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 108])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac507aba-8f76-47c6-bd22-6dff420215b2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2811064c-b3b2-438e-8e0c-aa9a3bfa436c, kind: Service { id: ac507aba-8f76-47c6-bd22-6dff420215b2 }, name: Name(\"external-dns-ac507aba-8f76-47c6-bd22-6dff420215b2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 139])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b39a7e8e-2dea-45da-9a7a-6977886585dc, kind: Service { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa }, name: Name(\"external-dns-b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 77])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: c108631e-f757-4a42-b510-28307153f426, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6ab9e3b5-f385-4e7e-9bc6-6c4f7a443782, kind: Service { id: c108631e-f757-4a42-b510-28307153f426 }, name: Name(\"nexus-c108631e-f757-4a42-b510-28307153f426\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 191])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6caaab71-9cdd-4faa-b082-987c9dd43604, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: ae27ee1d-0f9e-4fca-b104-9220b85a9848, kind: Service { id: 6caaab71-9cdd-4faa-b082-987c9dd43604 }, name: Name(\"nexus-6caaab71-9cdd-4faa-b082-987c9dd43604\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 147])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 37d6282c-2511-4c34-b419-e68b280bef44, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 97294434-1769-4a97-a97a-20d887e8c50a, kind: Service { id: 37d6282c-2511-4c34-b419-e68b280bef44 }, name: Name(\"nexus-37d6282c-2511-4c34-b419-e68b280bef44\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 88])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: abb1e589-7222-4a55-90ee-b0be940b410b, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: f864e092-4084-4307-8b83-2fd99a97943d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: fce1706b-b90a-4a5d-86b7-20320feb3342, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 32f00476-bb7b-4627-82ad-33c437d3a384, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: e736985c-90fa-417e-89f0-76ea8e60041f, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5baef1a-4fec-4af8-b323-da15f9e7b116, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 21592584-d863-4125-9acf-64a36ff982d8, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:27.916538341Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:27.917089756Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:27.917103181Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:27.939852763Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:27.942120552Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:27.959608294Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:27.959676376Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:27.978811784Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:27.978869572Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:27.996401426Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:27.996461829Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:28.017688972Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:28.017751578Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:28.035273328Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:28.035324666Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:28.052075995Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:28.052125955Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:28.072863383Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:28.072902955Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:28.091931405Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:28.091981215Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:28.109484365Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:28.109542724Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:28.129707504Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:28.12975317Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:28.147498969Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:28.147548703Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:28.166239487Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:28.166283296Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:28.184146631Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:28.184202216Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:28.201399249Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:28.201447475Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:28.220908015Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:28.220966966Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:28.238794773Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:28.238851257Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:28.25759673Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:28.257652352Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:28.275378093Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:28.275433707Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:28.293376001Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Reading ledger from /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:28.293425793Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:28.293730447Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:28.293874016Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:28.386445345Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:28.386523683Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:28.38653843Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:28.386545283Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:28.386553589Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:28.386626491Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:28.386633261Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:28.386641614Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:28.386646442Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:28.386653164Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:28.386668305Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:28.386681392Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:28.386690008Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:28.386707071Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:28.386713173Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:28.386721434Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:28.386726056Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:28.386737052Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:28.38674204Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:28.386807769Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"4db36c9b-4c4b-4ea8-9f65-da9e29204e66","remote_addr":"[fd00:1122:3344:101::1]:63238","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\", TimeNotSynchronized), (\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\", TimeNotSynchronized), (\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\", TimeNotSynchronized), (\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\", TimeNotSynchronized), (\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\", TimeNotSynchronized), (\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\", TimeNotSynchronized), (\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\", TimeNotSynchronized), (\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\", TimeNotSynchronized), (\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\", TimeNotSynchronized), (\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\", TimeNotSynchronized), (\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\", TimeNotSynchronized), (\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\", TimeNotSynchronized), (\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\", TimeNotSynchronized), (\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\", TimeNotSynchronized), (\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\", TimeNotSynchronized), (\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\", TimeNotSynchronized), (\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\", TimeNotSynchronized), (\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\", TimeNotSynchronized), (\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\", TimeNotSynchronized)]","latency_us":468724,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:28.386971318Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"21.872371278s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4db36c9b-4c4b-4ea8-9f65-da9e29204e66\", \"content-length\": \"124\", \"date\": \"Wed, 10 Jul 2024 23:57:27 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4db36c9b-4c4b-4ea8-9f65-da9e29204e66\" }"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:28.608042846Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:64962"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:28.608113094Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"ae6245c8-15e9-4956-8cc0-ca7b403d9d42","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:64962","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":40,"response_code":"200"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:29.562145986Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:423","component":"producer-server","component":"MetricsManager","sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"oximeter/producer/src/lib.rs:423","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"39.318014229s"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 89504e37-0b08-442d-ace9-40ca97c7ca33, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5ae75945-01ab-454a-af38-a5dd1d09d270, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a390c63c-a68b-4301-b3c7-797628a3170a, kind: Service { id: 5ae75945-01ab-454a-af38-a5dd1d09d270 }, name: Name(\"ntp-5ae75945-01ab-454a-af38-a5dd1d09d270\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 108])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: ac507aba-8f76-47c6-bd22-6dff420215b2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2811064c-b3b2-438e-8e0c-aa9a3bfa436c, kind: Service { id: ac507aba-8f76-47c6-bd22-6dff420215b2 }, name: Name(\"external-dns-ac507aba-8f76-47c6-bd22-6dff420215b2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 139])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: b39a7e8e-2dea-45da-9a7a-6977886585dc, kind: Service { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa }, name: Name(\"external-dns-b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 77])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: c108631e-f757-4a42-b510-28307153f426, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6ab9e3b5-f385-4e7e-9bc6-6c4f7a443782, kind: Service { id: c108631e-f757-4a42-b510-28307153f426 }, name: Name(\"nexus-c108631e-f757-4a42-b510-28307153f426\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 191])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6caaab71-9cdd-4faa-b082-987c9dd43604, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: ae27ee1d-0f9e-4fca-b104-9220b85a9848, kind: Service { id: 6caaab71-9cdd-4faa-b082-987c9dd43604 }, name: Name(\"nexus-6caaab71-9cdd-4faa-b082-987c9dd43604\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 147])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 37d6282c-2511-4c34-b419-e68b280bef44, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 97294434-1769-4a97-a97a-20d887e8c50a, kind: Service { id: 37d6282c-2511-4c34-b419-e68b280bef44 }, name: Name(\"nexus-37d6282c-2511-4c34-b419-e68b280bef44\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 88])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: abb1e589-7222-4a55-90ee-b0be940b410b, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: f864e092-4084-4307-8b83-2fd99a97943d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: fce1706b-b90a-4a5d-86b7-20320feb3342, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 32f00476-bb7b-4627-82ad-33c437d3a384, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External } } } }, OmicronZoneConfig { id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External } } } }, OmicronZoneConfig { id: e736985c-90fa-417e-89f0-76ea8e60041f, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5baef1a-4fec-4af8-b323-da15f9e7b116, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 21592584-d863-4125-9acf-64a36ff982d8, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.320774282Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.321340809Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.321356812Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 89504e37-0b08-442d-ace9-40ca97c7ca33, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.341355533Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.341412749Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 5681e7b4-0cf4-43e7-8c17-55b08b3eaa0d, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.35875251Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.358814358Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 53f5ee67-e6ff-49bd-88ae-d78f8180817c, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.378264529Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.378311415Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 08c14dbf-bb82-4151-bf37-eb74f12a90ee, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.395686403Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.395740844Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 31df8de9-1afd-4bb7-abe2-60daf0674d5e, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.41497936Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.415027754Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3d5a475d-f26b-4201-bcf1-199b682d3f89, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.43249636Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.432611068Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 49e6eea4-7bb4-41e1-bdad-3af89cf5dce4, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.45120047Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.451265008Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87f22eac-e992-48b8-979d-a5ce8a11b65f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.470428967Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.470483286Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: ac507aba-8f76-47c6-bd22-6dff420215b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.489641377Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.489765733Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.508054274Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.5081121Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f9fe7fe2-34ab-46fa-bcc4-4372f46845e2, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.526495177Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.528458558Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82a3c7f1-4f02-496a-9f3b-10a17fe07c27, dataset_name: DatasetName { pool_name: ZpoolName { id: 2ff702b9-0076-4c58-8af6-039be08bef86 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.544807269Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.544923678Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 87d55ca5-519a-4c9b-8a2e-01cb8a9bde70, dataset_name: DatasetName { pool_name: ZpoolName { id: 960eb39e-b913-4eb7-b0fb-372e6f629bcb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.56240455Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.562461615Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 94d92899-ee11-4114-a2a6-3abd344d4a6d, dataset_name: DatasetName { pool_name: ZpoolName { id: f91400c1-df8c-4cdb-b4f6-cb47c0d105a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.581107306Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.581171301Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 7e2165bb-d436-4e41-8a1a-ca14c1ee582b, dataset_name: DatasetName { pool_name: ZpoolName { id: d29abc28-1ae7-4192-8340-9766fa8fec22 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.599287204Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.599331698Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e736985c-90fa-417e-89f0-76ea8e60041f, dataset_name: DatasetName { pool_name: ZpoolName { id: c75afc92-cd90-403e-b6d4-baabf826a6f6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.618184587Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.618263544Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5baef1a-4fec-4af8-b323-da15f9e7b116, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f9779f-728b-405d-a726-810b70de4ab6 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.637149073Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.637197645Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 21592584-d863-4125-9acf-64a36ff982d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 2bc49789-2398-4c5f-be70-ff11292cbf60 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.655328054Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.655379871Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 37e3fcef-9a61-46a7-acea-3184c5e20a7e, dataset_name: DatasetName { pool_name: ZpoolName { id: 5a2ec449-297e-4e77-989a-dd421ce4d4aa (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.674291447Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.674362257Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9d5926a5-7f92-4354-a5d0-796448d8cf77, dataset_name: DatasetName { pool_name: ZpoolName { id: 4cab71ff-67df-44db-adca-8ef81d5bc209 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.692141989Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Reading ledger from /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.692205412Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.692543095Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.692684482Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.793898237Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.852980812Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.85334719Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.853383584Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.853414654Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.853455538Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.853489027Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.853507814Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.866134746Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:50.866267798Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.027705828Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.02800239Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.028021684Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.040301208Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.040528826Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.064950067Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.065018655Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.065025536Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.065031582Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Configuring new Omicron zone: oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.066664577Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.071179624Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.076017123Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.081141511Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.085969309Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.090870553Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.095517109Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.236913275Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.242070698Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.246606467Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.250904187Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.255311414Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.260102553Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.266295527Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Configuring new Omicron zone: oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.271601443Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.276681854Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.282368411Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.288632193Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.293994384Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.299517012Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.305223259Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.312044559Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.317379671Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.323728879Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.329741861Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.335568331Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.342371957Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.349707043Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.412723432Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.481754701Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.546280464Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.603798998Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.66067534Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.664321299Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.664338052Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.785364745Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:478","port":"Port { inner: PortInner(PortData { name: \"opte1\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 157, 147]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte1\" }) }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.785436698Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1200","zone_type":"nexus"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.789390968Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.789405904Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.91409835Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:478","port":"Port { inner: PortInner(PortData { name: \"opte2\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 230, 88]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte2\" }) }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.914184873Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1200","zone_type":"nexus"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.916902334Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:51.916927599Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:52.013303476Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:478","port":"Port { inner: PortInner(PortData { name: \"opte3\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 158, 191]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte3\" }) }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:52.013371436Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1200","zone_type":"nexus"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:52.017317629Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:52.01735708Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:52.121184013Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:478","port":"Port { inner: PortInner(PortData { name: \"opte4\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 211, 77]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte4\" }) }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:52.121254092Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1200","zone_type":"external_dns"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:52.124535074Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:52.124571149Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:52.216098153Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:478","port":"Port { inner: PortInner(PortData { name: \"opte5\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 139, 139]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte5\" }) }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:52.216163939Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1200","zone_type":"external_dns"}
{"msg":"no nat entry found for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:52.229657083Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"}
{"msg":"creating nat entry for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:52.229673695Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"}
{"msg":"no nat entry found for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:52.229983449Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"}
{"msg":"creating nat entry for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:52.230000118Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"}
{"msg":"no nat entry found for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:52.232293628Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"}
{"msg":"creating nat entry for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:52.232313137Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"}
{"msg":"no nat entry found for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:52.232556187Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"}
{"msg":"creating nat entry for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:52.232578002Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"}
{"msg":"no nat entry found for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:52.233224219Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"}
{"msg":"creating nat entry for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:52.233237485Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"}
{"msg":"creation of nat entry successful for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:52.244806493Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:52.247255897Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"creation of nat entry successful for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:52.247311942Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"}
{"msg":"creation of nat entry successful for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:52.249763371Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:52.253798736Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:52.25381683Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"creation of nat entry successful for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:52.291909896Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:52.295301855Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"creation of nat entry successful for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:52.295462357Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"}
{"msg":"Configuring new Omicron zone: oxz_nexus_c108631e-f757-4a42-b510-28307153f426","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:52.356417487Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:52.389986918Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Configuring new Omicron zone: oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:52.390408197Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:52.398268003Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_nexus_c108631e-f757-4a42-b510-28307153f426","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:52.404654438Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Configuring new Omicron zone: oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:52.412120324Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:52.45088449Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:52.45767815Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:52.464041137Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Configuring new Omicron zone: oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:52.471031323Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:52.587509133Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Profile for oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService14\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::12\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_2ff702b9-0076-4c58-8af6-039be08bef86/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::12\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"b06a0521-8130-47ef-9ac2-790de2c7e6f4\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:54.941182331Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:54.950561996Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService16\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::16\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_c75afc92-cd90-403e-b6d4-baabf826a6f6/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::16\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"7dc28ddf-bdcd-4a4a-bc2e-bc703c4eb3ef\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:54.954227427Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService18\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::15\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_d29abc28-1ae7-4192-8340-9766fa8fec22/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::15\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"2bb714f0-16dd-4d8a-b723-39aae2e06e4d\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:54.954488955Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService21\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::14\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_f91400c1-df8c-4cdb-b4f6-cb47c0d105a8/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::14\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"468f7697-462c-43ac-9821-4ebbcb5d56a5\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:54.955724622Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService23\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::19\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_5a2ec449-297e-4e77-989a-dd421ce4d4aa/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::19\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"3008fe43-6511-4eeb-b976-5408d4d4dec6\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:54.955981476Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService10\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::17\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_e6f9779f-728b-405d-a726-810b70de4ab6/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::17\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"9dd6205e-6ce0-43fa-9292-c50fa3f5ee8b\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:54.956181259Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService12\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::18\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_2bc49789-2398-4c5f-be70-ff11292cbf60/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::18\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"fe9214e1-fc5d-4aa4-8f85-1c53ddbdea13\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:54.95634477Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService13\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::13\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_960eb39e-b913-4eb7-b0fb-372e6f629bcb/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::13\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"a392e14c-67a7-40e4-92b5-13f77cb63810\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:54.956499807Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:54.956504738Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:54.963506468Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:54.969747755Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:54.97554601Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:54.98209659Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:54.988862766Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:54.99645068Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService22\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::d\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/oximeter\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"id\" value=\"abb1e589-7222-4a55-90ee-b0be940b410b\"/>\n        <propval type=\"astring\" name=\"address\" value=\"[fd00:1122:3344:101::d]:12223\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:55.086050834Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:57:55.086358156Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\") failed: Property not found. retry in 66.144979ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:55.271480414Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\") failed: Property not found. retry in 67.694673ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:55.285010865Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\") failed: Property not found. retry in 74.250826ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:55.296354231Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\") failed: Property not found. retry in 51.719746ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:55.307484546Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\") failed: Property not found. retry in 41.252535ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:55.318514034Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\") failed: Property not found. retry in 72.404189ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:55.32848084Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\") failed: Property not found. retry in 44.944739ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:55.339158706Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\") failed: Property not found. retry in 54.235611ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:55.349078741Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\") failed: Property not found. retry in 70.030717ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:55.368014871Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\") failed: Property not found. retry in 107.260657ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:57:55.475452766Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\") failed: Property not found. retry in 86.60256ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:00.732371205Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\") failed: Property not found. retry in 96.577133ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:00.764780552Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\") failed: Property not found. retry in 58.183845ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:00.799039964Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\") failed: Property not found. retry in 26.771457ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:00.831049017Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\") failed: Property not found. retry in 121.103276ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:00.860173294Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\") failed: Property not found. retry in 58.171588ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:00.881202523Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\") failed: Property not found. retry in 277.150395ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:00.908612013Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\") failed: Property not found. retry in 77.536624ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:00.945676948Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\") failed: Property not found. retry in 258.825348ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:00.974730882Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\") failed: Property not found. retry in 126.350952ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:00.993057707Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\") failed: Property not found. retry in 243.087956ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:01.012824995Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\") failed: Property not found. retry in 228.441622ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:01.033029915Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\") failed: Property not found. retry in 123.798076ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:01.050398154Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\") failed: Property not found. retry in 291.33267ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:01.06654207Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\") failed: Property not found. retry in 199.815065ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:01.082908477Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\") failed: Property not found. retry in 100.469731ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:01.098341145Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\") failed: Property not found. retry in 443.846773ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:01.138179849Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\") failed: Property not found. retry in 164.017489ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:01.190761621Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\") failed: Property not found. retry in 449.345539ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:01.208025218Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\") failed: Property not found. retry in 378.206437ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:01.227630992Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\") failed: Property not found. retry in 471.55776ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:01.251323729Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\") failed: Property not found. retry in 325.665102ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:01.275827256Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\") failed: Property not found. retry in 357.325878ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:01.303328635Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\") failed: Property not found. retry in 427.496029ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:01.320431861Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\") failed: Property not found. retry in 540.645318ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:01.372462927Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\") failed: Property not found. retry in 469.685118ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:01.389865256Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\") failed: Property not found. retry in 1.151880486s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:01.606833336Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\") failed: Property not found. retry in 617.481204ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:01.635732161Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\") failed: Property not found. retry in 610.026155ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:01.668011281Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\") failed: Property not found. retry in 929.149434ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:01.697130968Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\") failed: Property not found. retry in 1.152742825s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:01.719095807Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\") failed: Property not found. retry in 1.119702996s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:01.743180348Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\") failed: Property not found. retry in 475.074668ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:01.771561727Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\") failed: Property not found. retry in 937.835425ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:01.877390608Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\") failed: Property not found. retry in 407.454449ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:01.931874791Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\") failed: Property not found. retry in 847.651401ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:02.261988451Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\") failed: Property not found. retry in 1.240254783s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:02.276781152Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\") failed: Property not found. retry in 1.065020056s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:02.296223234Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\") failed: Property not found. retry in 736.828891ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:02.353945552Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\") failed: Property not found. retry in 1.150893748s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:02.640199435Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\") failed: Property not found. retry in 1.31007587s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:02.772426943Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\") failed: Property not found. retry in 915.287488ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:02.829211811Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\") failed: Property not found. retry in 937.772289ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:02.876580626Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\") failed: Property not found. retry in 896.681931ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:02.889073902Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService20\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::11\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/pantry\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::11\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"17000\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:02.917077606Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService15\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::10\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/pantry\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::10\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"17000\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:02.977915958Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService19\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::f\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/pantry\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::f\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"17000\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:03.024666447Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:03.024723327Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:03.05161892Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:03.066966257Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\") failed: Property not found. retry in 591.844157ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:03.108841472Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\") failed: Property not found. retry in 704.838738ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:03.126593035Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService17\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::1a\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_4cab71ff-67df-44db-adca-8ef81d5bc209/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::1a\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"4fea7780-fcb1-4840-a0a9-83c520fc6fbf\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:03.306213539Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:03.306495162Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\") failed: Property not found. retry in 64.979436ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:03.332370804Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\") failed: Property not found. retry in 70.095392ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:03.350143117Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\") failed: Property not found. retry in 1.203310392s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:03.373119617Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\") failed: Property not found. retry in 46.282062ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:03.383731192Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\") failed: Property not found. retry in 117.310069ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:03.407510721Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\") failed: Property not found. retry in 82.144599ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:03.441968386Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\") failed: Property not found. retry in 80.856025ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:03.484754101Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\") failed: Property not found. retry in 1.253252314s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:03.535494023Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\") failed: Property not found. retry in 187.523153ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:03.551822721Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\") failed: Property not found. retry in 231.055733ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:03.565953844Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\") failed: Property not found. retry in 184.088134ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:03.580694915Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\") failed: Property not found. retry in 45.734062ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:03.619522437Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\") failed: Property not found. retry in 124.706085ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:03.681975536Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\") failed: Property not found. retry in 1.370346109s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:03.71821802Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\") failed: Property not found. retry in 213.682454ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:03.759301247Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\") failed: Property not found. retry in 848.62911ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:03.775007968Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\") failed: Property not found. retry in 470.089402ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:03.793484909Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\") failed: Property not found. retry in 1.403184463s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:03.80916716Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\") failed: Property not found. retry in 766.6908ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:03.82485118Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\") failed: Property not found. retry in 871.839316ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:03.840943945Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\") failed: Property not found. retry in 106.352026ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:03.858105862Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\") failed: Property not found. retry in 456.480046ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:03.873625361Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\") failed: Property not found. retry in 826.358141ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:03.888553933Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\") failed: Property not found. retry in 227.911462ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:03.981482544Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\") failed: Property not found. retry in 697.957294ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:03.996149663Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\") failed: Property not found. retry in 835.29802ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:04.097401685Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\") failed: Property not found. retry in 617.678646ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:04.226762612Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\") failed: Property not found. retry in 1.117221725s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:04.282819532Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService27\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::9\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value=\"vopte4\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"172.30.1.1\"/>\n        <propval type=\"astring\" name=\"ip\" value=\"172.30.1.6\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/external_dns\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value=\"[fd00:1122:3344:101::9]:5353\"/>\n        <propval type=\"astring\" name=\"dns_address\" value=\"172.30.1.6:53\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:04.318321058Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:04.318624031Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\") failed: Property not found. retry in 697.499466ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:04.3451948Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\") failed: Property not found. retry in 830.538008ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:04.590219965Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\") failed: Property not found. retry in 35.946287ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:04.604196657Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\") failed: Property not found. retry in 1.412571195s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:04.61461636Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\") failed: Property not found. retry in 1.321886489s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:04.636046614Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\") failed: Property not found. retry in 82.45126ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:04.652972687Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\") failed: Property not found. retry in 841.202794ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:04.766267259Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\") failed: Property not found. retry in 1.387817271s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:04.788870785Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\") failed: Property not found. retry in 1.190916525s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:04.806978062Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\") failed: Property not found. retry in 152.638765ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:04.827724307Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\") failed: Property not found. retry in 537.895123ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:04.849261721Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\") failed: Property not found. retry in 1.267530413s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:04.870367451Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService28\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::8\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value=\"vopte5\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"172.30.1.1\"/>\n        <propval type=\"astring\" name=\"ip\" value=\"172.30.1.5\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/external_dns\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value=\"[fd00:1122:3344:101::8]:5353\"/>\n        <propval type=\"astring\" name=\"dns_address\" value=\"172.30.1.5:53\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:04.930380024Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:04.932218784Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\") failed: Property not found. retry in 969.219171ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:04.982334579Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\") failed: Property not found. retry in 436.183965ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:05.003277557Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\") failed: Property not found. retry in 612.451675ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:05.060763822Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\") failed: Property not found. retry in 521.863379ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:05.10929836Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\") failed: Property not found. retry in 1.418656211s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:05.229436847Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\") failed: Property not found. retry in 44.726223ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:05.272881759Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\") failed: Property not found. retry in 118.982698ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:05.330227523Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\") failed: Property not found. retry in 782.832169ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:05.401203725Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\") failed: Property not found. retry in 852.301215ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:05.437651512Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\") failed: Property not found. retry in 789.621501ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:05.494864944Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\") failed: Property not found. retry in 279.351213ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:05.519492911Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\") failed: Property not found. retry in 464.411622ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:05.536567457Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\") failed: Property not found. retry in 1.475762876s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:05.632721063Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\") failed: Property not found. retry in 1.419418956s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:05.654422819Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\") failed: Property not found. retry in 1.376217175s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:05.690634443Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\") failed: Property not found. retry in 269.92386ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:05.81358068Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\") failed: Property not found. retry in 526.500402ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:05.964180147Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\") failed: Property not found. retry in 1.197197251s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:05.976835335Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\") failed: Property not found. retry in 795.019494ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:06.008276737Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\") failed: Property not found. retry in 647.803603ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:06.018772601Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\") failed: Property not found. retry in 888.348901ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:06.039901501Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\") failed: Property not found. retry in 826.916295ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:06.094556846Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\") failed: Property not found. retry in 555.385655ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:06.149118409Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\") failed: Property not found. retry in 1.312148058s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:06.188398486Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\") failed: Property not found. retry in 992.573437ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:06.199048092Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\") failed: Property not found. retry in 1.366651279s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:06.297964118Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\") failed: Property not found. retry in 1.419252393s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:06.313931922Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\") failed: Property not found. retry in 911.688759ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:06.506134952Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\") failed: Property not found. retry in 964.490214ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:06.668821362Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\") failed: Property not found. retry in 1.156200513s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:06.682260748Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\") failed: Property not found. retry in 802.096046ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:06.718147418Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\") failed: Property not found. retry in 1.43479464s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:06.817082215Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService25\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::b\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value=\"vopte1\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"172.30.2.1\"/>\n        <propval type=\"astring\" name=\"ip\" value=\"172.30.2.6\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/nexus\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:06.914435113Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService26\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::c\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value=\"vopte2\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"172.30.2.1\"/>\n        <propval type=\"astring\" name=\"ip\" value=\"172.30.2.7\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/nexus\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:06.914499584Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_nexus_c108631e-f757-4a42-b510-28307153f426:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService24\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::a\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value=\"vopte3\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"172.30.2.1\"/>\n        <propval type=\"astring\" name=\"ip\" value=\"172.30.2.5\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/nexus\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:06.914780937Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:06.914825118Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:06.923182289Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:06.930309136Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\") failed: Property not found. retry in 704.831123ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:06.947990718Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\") failed: Property not found. retry in 1.190559942s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:06.960824218Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\") failed: Property not found. retry in 1.016463504s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:07.082041806Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\") failed: Property not found. retry in 1.201952777s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:07.094823445Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\") failed: Property not found. retry in 607.481593ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:07.122941161Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\") failed: Property not found. retry in 31.86445ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:07.151047057Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\") failed: Property not found. retry in 71.734717ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:07.164767196Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\") failed: Property not found. retry in 32.147499ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:07.179021621Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\") failed: Property not found. retry in 720.962636ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:07.191490827Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\") failed: Property not found. retry in 56.416469ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:07.203856001Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\") failed: Property not found. retry in 1.223853797s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:07.219566242Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\") failed: Property not found. retry in 129.519374ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:07.235308064Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\") failed: Property not found. retry in 75.229461ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:07.262000734Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\") failed: Property not found. retry in 174.003857ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:07.279001309Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\") failed: Property not found. retry in 207.604603ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:07.372703438Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\") failed: Property not found. retry in 189.079484ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:07.455914653Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\") failed: Property not found. retry in 561.502145ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:07.491310708Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\") failed: Property not found. retry in 961.074994ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:07.529824807Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\") failed: Property not found. retry in 1.486422995s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:07.554106061Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\") failed: Property not found. retry in 952.421055ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:07.581325607Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\") failed: Property not found. retry in 512.158745ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:07.635574567Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\") failed: Property not found. retry in 508.411247ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:07.64875727Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\") failed: Property not found. retry in 481.343943ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:07.664959837Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\") failed: Property not found. retry in 1.366642653s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:07.677753366Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\") failed: Property not found. retry in 1.330849196s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:07.689833837Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\") failed: Property not found. retry in 533.302495ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:07.744594693Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\") failed: Property not found. retry in 1.202691493s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:07.755777314Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\") failed: Property not found. retry in 1.319565584s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:07.853911632Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\") failed: Property not found. retry in 589.95577ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:07.926089027Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\") failed: Property not found. retry in 881.103376ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:08.384014487Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\") failed: Property not found. retry in 1.44624621s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:08.395090688Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\") failed: Property not found. retry in 834.097328ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:08.406165028Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\") failed: Property not found. retry in 878.250311ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:08.416531931Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\") failed: Property not found. retry in 776.377091ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:08.433770979Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\") failed: Property not found. retry in 1.121484188s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:08.444621507Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\") failed: Property not found. retry in 1.214420211s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:08.454267322Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\") failed: Property not found. retry in 508.359095ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:08.466506365Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\") failed: Property not found. retry in 1.22104716s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:08.478192162Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\") failed: Property not found. retry in 893.107239ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:08.530699013Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\") failed: Property not found. retry in 1.399138086s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:08.542058146Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\") failed: Property not found. retry in 1.110891016s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:08.553023326Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\") failed: Property not found. retry in 1.017125089s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:08.563563991Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService11\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::e\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/clickhouse\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::e\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"8123\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:3::1\" />\n            <value_node value=\"fd00:1122:3344:2::1\" />\n            <value_node value=\"fd00:1122:3344:1::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:08.670647845Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:08.671291922Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\") failed: Property not found. retry in 56.484358ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:08.94238144Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\") failed: Property not found. retry in 926.219638ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:08.972547778Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\") failed: Property not found. retry in 966.881381ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:09.225843893Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\") failed: Property not found. retry in 1.103709967s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:09.250182328Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\") failed: Property not found. retry in 58.517778ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:09.651793406Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\") failed: Property not found. retry in 1.43595776s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:09.896356696Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\") failed: Property not found. retry in 868.829114ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:10.000676501Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\") failed: Property not found. retry in 1.490564893s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:10.052192541Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\") failed: Property not found. retry in 912.480965ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:10.148902313Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\") failed: Property not found. retry in 1.042611517s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:10.247514906Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\") failed: Property not found. retry in 661.101816ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:10.296472778Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\") failed: Property not found. retry in 1.074095088s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:10.395123851Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\") failed: Property not found. retry in 1.405719439s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:10.444043933Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\") failed: Property not found. retry in 1.044891786s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:10.500448497Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\") failed: Property not found. retry in 564.335234ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:10.58259274Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\") failed: Property not found. retry in 1.250513161s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:10.693601958Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\") failed: Property not found. retry in 1.266062135s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:10.722923547Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\") failed: Property not found. retry in 683.961071ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:10.733958386Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\") failed: Property not found. retry in 573.645271ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:10.763139344Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\") failed: Property not found. retry in 1.2093901s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:10.784371075Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\") failed: Property not found. retry in 863.002433ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:10.822126816Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\") failed: Property not found. retry in 700.887651ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:10.866776081Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\") failed: Property not found. retry in 843.23079ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:10.982215017Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\") failed: Property not found. retry in 106.492773ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:11.042974208Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\") failed: Property not found. retry in 683.144121ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:11.052065517Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\") failed: Property not found. retry in 801.068996ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:11.114421265Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\") failed: Property not found. retry in 838.029603ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:11.12321188Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\") failed: Property not found. retry in 987.785111ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:11.261290712Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\") failed: Property not found. retry in 551.512618ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:11.270222549Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\") failed: Property not found. retry in 898.053041ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:11.333906942Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\") failed: Property not found. retry in 1.277157008s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:11.345998683Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\") failed: Property not found. retry in 755.396602ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:11.354821479Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\") failed: Property not found. retry in 1.013553759s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:11.42845784Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\") failed: Property not found. retry in 772.923634ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:11.477087579Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\") failed: Property not found. retry in 1.371388531s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:11.552035215Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\") failed: Property not found. retry in 1.24385724s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:11.560065022Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\") failed: Property not found. retry in 1.21306501s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:11.576087426Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\") failed: Property not found. retry in 863.822033ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:11.693430162Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\") failed: Property not found. retry in 1.23131158s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:11.743057221Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\") failed: Property not found. retry in 941.312692ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:11.829766515Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\") failed: Property not found. retry in 878.224994ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:11.837405778Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\") failed: Property not found. retry in 1.495957883s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:11.85964708Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\") failed: Property not found. retry in 1.311252822s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:11.926025392Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\") failed: Property not found. retry in 573.990109ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:11.954436241Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\") failed: Property not found. retry in 542.366597ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:11.970056321Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\") failed: Property not found. retry in 630.27583ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:11.997176376Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\") failed: Property not found. retry in 1.431010949s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:12.005028951Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\") failed: Property not found. retry in 1.341843094s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:12.120527867Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\") failed: Property not found. retry in 624.003227ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:12.244703678Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\") failed: Property not found. retry in 1.182188302s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:12.260482189Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\") failed: Property not found. retry in 1.31168749s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:12.269632509Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\") failed: Property not found. retry in 1.191680509s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:12.452368827Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\") failed: Property not found. retry in 1.034589272s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:12.522465989Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\") failed: Property not found. retry in 1.061974298s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:12.538285451Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\") failed: Property not found. retry in 1.247441613s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:12.566865562Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\") failed: Property not found. retry in 580.566168ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:12.631567556Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\") failed: Property not found. retry in 910.218138ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:12.640109549Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\") failed: Property not found. retry in 1.115107807s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:12.725080783Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\") failed: Property not found. retry in 932.212922ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:12.779856478Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\") failed: Property not found. retry in 944.174299ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:12.797079996Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\") failed: Property not found. retry in 1.24716754s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:12.812360402Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\") failed: Property not found. retry in 1.161856062s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:12.877347989Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\") failed: Property not found. retry in 630.880686ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:12.933027715Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\") failed: Property not found. retry in 1.390425405s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:12.982697445Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\") failed: Property not found. retry in 751.70446ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:13.221866836Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\") failed: Property not found. retry in 696.503147ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:13.246608825Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\") failed: Property not found. retry in 1.44281711s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:13.364961453Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\") failed: Property not found. retry in 1.378473607s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:13.44472196Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\") failed: Property not found. retry in 1.1351503s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:13.453588697Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\") failed: Property not found. retry in 977.523722ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:13.470871485Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\") failed: Property not found. retry in 1.184216115s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:13.560896164Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\") failed: Property not found. retry in 1.481351601s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:13.570142494Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\") failed: Property not found. retry in 1.030533394s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:13.579147692Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\") failed: Property not found. retry in 1.369631445s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:13.5899998Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\") failed: Property not found. retry in 1.251433467s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:13.609856456Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\") failed: Property not found. retry in 1.294910526s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:13.654170678Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\") failed: Property not found. retry in 1.214536238s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:13.722786965Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\") failed: Property not found. retry in 526.211531ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:13.751140463Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\") failed: Property not found. retry in 1.163947692s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:13.82532274Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\") failed: Property not found. retry in 756.652444ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:13.851148641Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\") failed: Property not found. retry in 1.456616177s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:13.954213632Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\") failed: Property not found. retry in 1.449139837s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:13.981777322Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\") failed: Property not found. retry in 926.207968ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:14.048712419Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\") failed: Property not found. retry in 745.739746ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:14.068770028Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\") failed: Property not found. retry in 537.479464ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:14.291040715Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\") failed: Property not found. retry in 964.162262ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:14.389256474Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\") failed: Property not found. retry in 1.316035268s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:14.462149307Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\") failed: Property not found. retry in 1.419802478s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:14.603445513Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\") failed: Property not found. retry in 517.539299ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:14.622610432Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\") failed: Property not found. retry in 1.181676078s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:14.63627254Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\") failed: Property not found. retry in 654.704098ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:14.757428668Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\") failed: Property not found. retry in 664.340938ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:14.821229072Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\") failed: Property not found. retry in 610.997994ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:14.835999123Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\") failed: Property not found. retry in 1.426381159s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:14.849969825Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\") failed: Property not found. retry in 1.046465252s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:14.864627984Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\") failed: Property not found. retry in 801.964442ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:14.882293086Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\") failed: Property not found. retry in 735.367585ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:14.952304338Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\") failed: Property not found. retry in 1.313623619s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:14.962366807Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\") failed: Property not found. retry in 852.816182ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:14.974363398Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\") failed: Property not found. retry in 1.212834855s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:14.987986486Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\") failed: Property not found. retry in 628.996386ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:15.00123065Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\") failed: Property not found. retry in 1.459751046s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:15.062612387Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\") failed: Property not found. retry in 1.130494914s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:15.152785238Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\") failed: Property not found. retry in 1.380634589s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:15.367222081Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\") failed: Property not found. retry in 564.907616ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:15.424235091Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\") failed: Property not found. retry in 1.094542574s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:15.436015039Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\") failed: Property not found. retry in 797.576296ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:15.447847038Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\") failed: Property not found. retry in 880.990299ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:15.458346162Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\") failed: Property not found. retry in 804.266102ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:15.496923442Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\") failed: Property not found. retry in 823.943922ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:15.64111009Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\") failed: Property not found. retry in 1.220534511s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:15.694232338Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\") failed: Property not found. retry in 939.716983ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:15.704998615Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\") failed: Property not found. retry in 1.129914732s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:15.791220331Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\") failed: Property not found. retry in 911.711693ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:15.828584198Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\") failed: Property not found. retry in 1.251303756s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:15.837225442Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\") failed: Property not found. retry in 1.224657334s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:15.921470698Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\") failed: Property not found. retry in 942.452774ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:15.998410855Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\") failed: Property not found. retry in 1.101838489s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:16.034829091Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\") failed: Property not found. retry in 966.545858ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:16.212559374Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\") failed: Property not found. retry in 1.356886578s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:16.257413882Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\") failed: Property not found. retry in 794.732189ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:16.286242486Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\") failed: Property not found. retry in 1.397354906s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:16.296058302Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\") failed: Property not found. retry in 563.2393ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:16.304779048Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\") failed: Property not found. retry in 504.910768ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:16.31416141Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\") failed: Property not found. retry in 607.038931ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:16.349473974Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\") failed: Property not found. retry in 836.55888ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:16.477438655Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\") failed: Property not found. retry in 887.896129ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:16.533588416Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\") failed: Property not found. retry in 1.375262246s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:16.543511814Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\") failed: Property not found. retry in 946.643048ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:16.656810146Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\") failed: Property not found. retry in 1.380341869s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:16.750757307Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\") failed: Property not found. retry in 533.559613ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:16.759331421Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\") failed: Property not found. retry in 1.179189936s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:16.829285452Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\") failed: Property not found. retry in 1.429985798s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:16.878334155Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\") failed: Property not found. retry in 1.480438382s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:16.926136335Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d\") failed: Property not found. retry in 1.332363045s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:16.935712999Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\") failed: Property not found. retry in 710.816135ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:16.949861873Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\") failed: Property not found. retry in 1.329911314s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:16.965763166Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\") failed: Property not found. retry in 997.695508ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:17.090799786Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\") failed: Property not found. retry in 930.417766ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:17.099135527Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\") failed: Property not found. retry in 1.388243351s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:17.146152018Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\") failed: Property not found. retry in 514.883886ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:17.15550959Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\") failed: Property not found. retry in 1.08382226s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:17.189276317Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\") failed: Property not found. retry in 974.273255ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:17.304594682Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\") failed: Property not found. retry in 616.43026ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:17.323060742Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\") failed: Property not found. retry in 1.101117976s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:17.445462094Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\") failed: Property not found. retry in 911.321739ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:17.61243157Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\") failed: Property not found. retry in 883.713046ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:17.623043815Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\") failed: Property not found. retry in 1.079022848s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:17.670439931Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b\") failed: Property not found. retry in 1.388226357s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:17.679248017Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\") failed: Property not found. retry in 890.253082ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:17.702804993Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\") failed: Property not found. retry in 1.033323222s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:17.929530719Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e\") failed: Property not found. retry in 932.468244ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:17.950134043Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\") failed: Property not found. retry in 750.407128ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:18.022012294Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\") failed: Property not found. retry in 1.304944104s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:18.040595827Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\") failed: Property not found. retry in 841.231536ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:18.099176914Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27\") failed: Property not found. retry in 1.431108627s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:18.143293744Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\") failed: Property not found. retry in 506.8786ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:18.317370147Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\") failed: Property not found. retry in 744.279084ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:18.327419836Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b\") failed: Property not found. retry in 537.195896ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:18.340425058Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116\") failed: Property not found. retry in 1.137881478s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:18.354055856Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d\") failed: Property not found. retry in 990.272353ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:18.418729229Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8\") failed: Property not found. retry in 876.443027ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:18.521809651Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\") failed: Property not found. retry in 1.281309408s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:18.535948144Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\") failed: Property not found. retry in 1.254723384s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:18.546165366Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\") failed: Property not found. retry in 583.878693ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:18.557077744Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\") failed: Property not found. retry in 1.422090049s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:18.611371985Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f\") failed: Property not found. retry in 973.093063ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:18.769400873Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\") failed: Property not found. retry in 1.347316973s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:18.788505681Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70\") failed: Property not found. retry in 511.145181ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:18.844722472Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\") failed: Property not found. retry in 612.711571ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:18.953711838Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\") failed: Property not found. retry in 509.294323ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:18.979650649Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\") failed: Property not found. retry in 558.768747ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:19.112014019Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\") failed: Property not found. retry in 1.164261106s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:19.158441454Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\") failed: Property not found. retry in 1.218279431s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:19.369116316Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\") failed: Property not found. retry in 1.226198061s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:19.509213789Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384\") failed: Property not found. retry in 886.849183ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:19.587536161Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\") failed: Property not found. retry in 746.859012ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:19.689135186Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\") failed: Property not found. retry in 648.491505ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:19.821144091Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\") failed: Property not found. retry in 1.354903345s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:19.842526654Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\") failed: Property not found. retry in 1.309607197s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:20.093829397Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\") failed: Property not found. retry in 1.228533285s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:20.167058194Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342\") failed: Property not found. retry in 572.945966ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:20.344295071Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\") failed: Property not found. retry in 927.027345ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:20.459807537Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\") failed: Property not found. retry in 599.184935ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:20.48857149Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\") failed: Property not found. retry in 1.146909756s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:20.607379412Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\") failed: Property not found. retry in 666.407543ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:20.757715906Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\") failed: Property not found. retry in 866.621607ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:21.104670959Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\") failed: Property not found. retry in 1.347425466s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:21.220695161Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\") failed: Property not found. retry in 993.862605ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:21.402219755Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77\") failed: Property not found. retry in 537.427204ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:21.420416563Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2\") failed: Property not found. retry in 1.451802048s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:21.436592649Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\") failed: Property not found. retry in 1.069589571s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:21.774424792Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c108631e-f757-4a42-b510-28307153f426\") failed: Property not found. retry in 1.265123825s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:22.004255292Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\") failed: Property not found. retry in 1.160249268s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:22.412927246Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\") failed: Property not found. retry in 1.374192515s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:22.583920216Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44\") failed: Property not found. retry in 1.288217471s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:22.863454374Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604\") failed: Property not found. retry in 508.216225ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:23.588831333Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\") failed: Property not found. retry in 882.019882ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:23.976046423Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\") failed: Property not found. retry in 892.405005ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:24.870600651Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\") failed: Property not found. retry in 1.353972864s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:25.774776453Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:26.34284863Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:423","component":"producer-server","component":"MetricsManager","sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"oximeter/producer/src/lib.rs:423","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"159.305603374s"}
{"msg":"wait for service Some(\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\") failed: Property not found. retry in 835.295979ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:27.139775326Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\") failed: Property not found. retry in 711.877399ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:27.985283411Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\") failed: Property not found. retry in 1.378183601s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:28.706747056Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\") failed: Property not found. retry in 1.048729992s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:30.09573653Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:30.874105544Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:40526"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:30.877614202Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"f417d4cb-2c59-476a-894d-d37cc7e50c21","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:40526","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":1782,"response_code":"200"}
{"msg":"wait for service Some(\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\") failed: Property not found. retry in 837.354705ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:31.15606404Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:31.451416792Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"}
{"msg":"wait for service Some(\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\") failed: Property not found. retry in 587.842733ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:32.004454356Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\") failed: Property not found. retry in 611.190393ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:32.603851384Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\") failed: Property not found. retry in 655.346807ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:33.225887419Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2\") failed: Property not found. retry in 1.203702787s","v":0,"name":"SledAgent","level":40,"time":"2024-07-10T23:58:33.892019223Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:35.126258305Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_external_dns_ac507aba-8f76-47c6-bd22-6dff420215b2"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:35.126316556Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_crucible_d5baef1a-4fec-4af8-b323-da15f9e7b116"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:35.126325206Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_clickhouse_f9fe7fe2-34ab-46fa-bcc4-4372f46845e2"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:35.126333246Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_crucible_21592584-d863-4125-9acf-64a36ff982d8"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:35.126351216Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_crucible_87d55ca5-519a-4c9b-8a2e-01cb8a9bde70"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:35.126355796Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_crucible_82a3c7f1-4f02-496a-9f3b-10a17fe07c27"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:35.126360296Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_crucible_pantry_fce1706b-b90a-4a5d-86b7-20320feb3342"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:35.126364546Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_nexus_6caaab71-9cdd-4faa-b082-987c9dd43604"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:35.126368866Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_crucible_e736985c-90fa-417e-89f0-76ea8e60041f"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:35.126373316Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_nexus_37d6282c-2511-4c34-b419-e68b280bef44"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:35.126377576Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_crucible_9d5926a5-7f92-4354-a5d0-796448d8cf77"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:35.126382156Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_crucible_7e2165bb-d436-4e41-8a1a-ca14c1ee582b"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:35.126386547Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_nexus_c108631e-f757-4a42-b510-28307153f426"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:35.126390736Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_crucible_pantry_f864e092-4084-4307-8b83-2fd99a97943d"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:35.126395287Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_external_dns_b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:35.126399437Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_crucible_pantry_32f00476-bb7b-4627-82ad-33c437d3a384"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:35.126403597Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_crucible_94d92899-ee11-4114-a2a6-3abd344d4a6d"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:35.126467567Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_oximeter_abb1e589-7222-4a55-90ee-b0be940b410b"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:35.126480917Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_crucible_37e3fcef-9a61-46a7-acea-3184c5e20a7e"}
{"msg":"Writing ledger to /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:35.126939342Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:35.12942498Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:35.134500975Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"ba0edac9-3084-4c87-99ec-14311ad6d1c2","remote_addr":"[fd00:1122:3344:101::1]:63238","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":44790394,"response_code":"204"}
{"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:35.150198506Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1244"}
{"msg":"Writing ledger to /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:35.150472269Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:35.15064864Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:35.154523693Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:681"}
{"msg":"Nexus address: [fd00:1122:3344:101::c]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:35.158587227Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:694"}
{"msg":"rack_network_config: RackNetworkConfigV2 {\n    bfd: [],\n    bgp: [],\n    infra_ip_first: 10.151.1.174,\n    infra_ip_last: 10.151.1.174,\n    ports: [\n        PortConfigV2 {\n            addresses: [\n                UplinkAddressConfig {\n                    address: V4(\n                        Ipv4Net {\n                            addr: 10.151.1.174,\n                            width: 24,\n                        },\n                    ),\n                    vlan_id: None,\n                },\n            ],\n            autoneg: false,\n            bgp_peers: [],\n            port: \"qsfp0\",\n            routes: [\n                RouteConfig {\n                    destination: V4(\n                        Ipv4Net {\n                            addr: 0.0.0.0,\n                            width: 0,\n                        },\n                    ),\n                    nexthop: 10.151.1.100,\n                    vlan_id: None,\n                },\n            ],\n            switch: Switch0,\n            uplink_port_fec: None,\n            uplink_port_speed: Speed40G,\n        },\n    ],\n    rack_subnet: Ipv6Net {\n        addr: fd00:1122:3344:100::,\n        width: 56,\n    },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:35.191337393Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:825"}
{"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"317421b3-2a67-4cbc-b6cb-c27e54dac834\", \"content-length\": \"173\", \"date\": \"Wed, 10 Jul 2024 23:58:35 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"0dcbb472-7cd1-4e42-9116-fc1cc27da573\\\"\", request_id: \"317421b3-2a67-4cbc-b6cb-c27e54dac834\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:35.747536741Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"}
{"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d6be8303-608b-4ca3-b645-44f68b9926fd\", \"content-length\": \"173\", \"date\": \"Wed, 10 Jul 2024 23:58:35 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"0dcbb472-7cd1-4e42-9116-fc1cc27da573\\\"\", request_id: \"d6be8303-608b-4ca3-b645-44f68b9926fd\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:36.188657698Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"}
{"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2a24b6bc-fd3e-4066-aa9a-aba4f9036489\", \"content-length\": \"173\", \"date\": \"Wed, 10 Jul 2024 23:58:35 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"0dcbb472-7cd1-4e42-9116-fc1cc27da573\\\"\", request_id: \"2a24b6bc-fd3e-4066-aa9a-aba4f9036489\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:36.683927704Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"}
{"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5426ee28-bba0-40d5-a23c-314e22c2e5d3\", \"content-length\": \"173\", \"date\": \"Wed, 10 Jul 2024 23:58:37 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"0dcbb472-7cd1-4e42-9116-fc1cc27da573\\\"\", request_id: \"5426ee28-bba0-40d5-a23c-314e22c2e5d3\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:37.198135855Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"}
{"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4bad09ab-6b36-4d95-affa-6899784cba89\", \"content-length\": \"173\", \"date\": \"Wed, 10 Jul 2024 23:58:37 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"0dcbb472-7cd1-4e42-9116-fc1cc27da573\\\"\", request_id: \"4bad09ab-6b36-4d95-affa-6899784cba89\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:37.730922649Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"}
{"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"943f21ad-e98a-4b08-8282-3354aea3778c\", \"content-length\": \"173\", \"date\": \"Wed, 10 Jul 2024 23:58:37 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"0dcbb472-7cd1-4e42-9116-fc1cc27da573\\\"\", request_id: \"943f21ad-e98a-4b08-8282-3354aea3778c\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:38.106886177Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"}
{"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1a6b3fd8-ce77-4cad-817e-80ee36bf9bd6\", \"content-length\": \"173\", \"date\": \"Wed, 10 Jul 2024 23:58:38 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"0dcbb472-7cd1-4e42-9116-fc1cc27da573\\\"\", request_id: \"1a6b3fd8-ce77-4cad-817e-80ee36bf9bd6\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:38.496992249Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"}
{"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7cc76aed-d6fd-441b-8772-45b7029c0d22\", \"content-length\": \"173\", \"date\": \"Wed, 10 Jul 2024 23:58:38 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"0dcbb472-7cd1-4e42-9116-fc1cc27da573\\\"\", request_id: \"7cc76aed-d6fd-441b-8772-45b7029c0d22\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:39.239533164Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"}
{"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c57693db-7af2-42af-80bd-206dba4ca8ba\", \"content-length\": \"173\", \"date\": \"Wed, 10 Jul 2024 23:58:39 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"0dcbb472-7cd1-4e42-9116-fc1cc27da573\\\"\", request_id: \"c57693db-7af2-42af-80bd-206dba4ca8ba\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:39.932405448Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"}
{"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bdad4530-7247-40e2-b3ad-7f200e3d80bb\", \"content-length\": \"173\", \"date\": \"Wed, 10 Jul 2024 23:58:40 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"0dcbb472-7cd1-4e42-9116-fc1cc27da573\\\"\", request_id: \"bdad4530-7247-40e2-b3ad-7f200e3d80bb\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:40.785900469Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"}
{"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"71d20c4a-0b81-492c-a39c-0ed0214ac928\", \"content-length\": \"173\", \"date\": \"Wed, 10 Jul 2024 23:58:40 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"0dcbb472-7cd1-4e42-9116-fc1cc27da573\\\"\", request_id: \"71d20c4a-0b81-492c-a39c-0ed0214ac928\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:41.390001338Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"}
{"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7ac9e69b-4e83-4b0c-923b-69e24bea9bf3\", \"content-length\": \"173\", \"date\": \"Wed, 10 Jul 2024 23:58:42 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"0dcbb472-7cd1-4e42-9116-fc1cc27da573\\\"\", request_id: \"7ac9e69b-4e83-4b0c-923b-69e24bea9bf3\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:42.417349111Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"}
{"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b8da6370-a030-4391-ae72-c3423c58b252\", \"content-length\": \"173\", \"date\": \"Wed, 10 Jul 2024 23:58:42 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"0dcbb472-7cd1-4e42-9116-fc1cc27da573\\\"\", request_id: \"b8da6370-a030-4391-ae72-c3423c58b252\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:43.125654553Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"}
{"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"015fc4d2-42b0-49d1-8b7c-190feb42bdc0\", \"content-length\": \"173\", \"date\": \"Wed, 10 Jul 2024 23:58:44 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"0dcbb472-7cd1-4e42-9116-fc1cc27da573\\\"\", request_id: \"015fc4d2-42b0-49d1-8b7c-190feb42bdc0\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:44.215796058Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"}
{"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3ebd96c4-32df-46a9-9e6c-a820d4af8971\", \"content-length\": \"173\", \"date\": \"Wed, 10 Jul 2024 23:58:45 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"0dcbb472-7cd1-4e42-9116-fc1cc27da573\\\"\", request_id: \"3ebd96c4-32df-46a9-9e6c-a820d4af8971\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:45.329820433Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"}
{"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"013191c2-d7ac-4e1f-900a-e04760cb71d6\", \"content-length\": \"173\", \"date\": \"Wed, 10 Jul 2024 23:58:47 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"0dcbb472-7cd1-4e42-9116-fc1cc27da573\\\"\", request_id: \"013191c2-d7ac-4e1f-900a-e04760cb71d6\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:47.468835013Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"}
{"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3383f70d-cfe0-49ba-9447-132040710513\", \"content-length\": \"173\", \"date\": \"Wed, 10 Jul 2024 23:58:48 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"0dcbb472-7cd1-4e42-9116-fc1cc27da573\\\"\", request_id: \"3383f70d-cfe0-49ba-9447-132040710513\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:48.577407199Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"}
{"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"99ccd90c-afc0-40e9-835b-54bd309ccfc3\", \"content-length\": \"173\", \"date\": \"Wed, 10 Jul 2024 23:58:50 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"0dcbb472-7cd1-4e42-9116-fc1cc27da573\\\"\", request_id: \"99ccd90c-afc0-40e9-835b-54bd309ccfc3\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:50.455462001Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"}
{"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"003dafaa-d17d-4802-a5db-50e0d20aa957\", \"content-length\": \"173\", \"date\": \"Wed, 10 Jul 2024 23:58:52 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"0dcbb472-7cd1-4e42-9116-fc1cc27da573\\\"\", request_id: \"003dafaa-d17d-4802-a5db-50e0d20aa957\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:52.434453852Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"}
{"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d63f4608-af0b-4bff-823c-113eae2ee28c\", \"content-length\": \"173\", \"date\": \"Wed, 10 Jul 2024 23:58:56 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"0dcbb472-7cd1-4e42-9116-fc1cc27da573\\\"\", request_id: \"d63f4608-af0b-4bff-823c-113eae2ee28c\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:58:56.630837385Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"}
{"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5dd3da80-b419-4f7e-80a2-a68a53a455f6\", \"content-length\": \"173\", \"date\": \"Wed, 10 Jul 2024 23:59:00 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"0dcbb472-7cd1-4e42-9116-fc1cc27da573\\\"\", request_id: \"5dd3da80-b419-4f7e-80a2-a68a53a455f6\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:00.712827024Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:00.910215979Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:58257"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:00.91031374Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"4924e2a2-38ff-4c04-aa83-11090c462e04","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:58257","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":56,"response_code":"200"}
{"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f7e7b055-e1d1-4fc0-a0eb-f25e64e3826d\", \"content-length\": \"173\", \"date\": \"Wed, 10 Jul 2024 23:59:03 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"0dcbb472-7cd1-4e42-9116-fc1cc27da573\\\"\", request_id: \"f7e7b055-e1d1-4fc0-a0eb-f25e64e3826d\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:03.296301836Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"}
{"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"81d16415-564d-4b3c-a622-aede6742feb3\", \"content-length\": \"173\", \"date\": \"Wed, 10 Jul 2024 23:59:07 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"0dcbb472-7cd1-4e42-9116-fc1cc27da573\\\"\", request_id: \"81d16415-564d-4b3c-a622-aede6742feb3\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:07.386053511Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"}
{"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"484ad635-53a2-48a6-b778-2e6c4f15f4ea\", \"content-length\": \"173\", \"date\": \"Wed, 10 Jul 2024 23:59:12 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"0dcbb472-7cd1-4e42-9116-fc1cc27da573\\\"\", request_id: \"484ad635-53a2-48a6-b778-2e6c4f15f4ea\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:12.460326559Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"}
{"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:14.510898538Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"sled-agent/src/nexus.rs:592","new_generation":"1","old_generation":"1"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:14.593070182Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:38055"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:14.594255064Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:14.682051909Z","hostname":"buskin","pid":1555,"uri":"/inventory","method":"GET","req_id":"71c4e221-2f28-45d5-b036-77d0d7436405","remote_addr":"[fd00:1122:3344:101::b]:38055","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":88877,"response_code":"200"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:14.684656218Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Reading ledger from /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:14.684731868Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:14.685198153Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:14.685479236Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"GET","req_id":"4673a796-a439-4d93-acd5-dabf9fee8758","remote_addr":"[fd00:1122:3344:101::b]:38055","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":1824,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:16.694533645Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:41976"}
{"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:16.697984142Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:725","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2811064c-b3b2-438e-8e0c-aa9a3bfa436c, kind: Service { id: ac507aba-8f76-47c6-bd22-6dff420215b2 }, name: Name(\"external-dns-ac507aba-8f76-47c6-bd22-6dff420215b2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 139])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b39a7e8e-2dea-45da-9a7a-6977886585dc, kind: Service { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa }, name: Name(\"external-dns-b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 77])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6ab9e3b5-f385-4e7e-9bc6-6c4f7a443782, kind: Service { id: c108631e-f757-4a42-b510-28307153f426 }, name: Name(\"nexus-c108631e-f757-4a42-b510-28307153f426\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 191])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 97294434-1769-4a97-a97a-20d887e8c50a, kind: Service { id: 37d6282c-2511-4c34-b419-e68b280bef44 }, name: Name(\"nexus-37d6282c-2511-4c34-b419-e68b280bef44\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 88])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ae27ee1d-0f9e-4fca-b104-9220b85a9848, kind: Service { id: 6caaab71-9cdd-4faa-b082-987c9dd43604 }, name: Name(\"nexus-6caaab71-9cdd-4faa-b082-987c9dd43604\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 147])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:16.698650089Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:16.699840882Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:16.699855713Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:16.699871073Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[]","port":"\"opte0\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:16.699884013Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:16.699901233Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:16.699929203Z","hostname":"buskin","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"61da4a4d-8107-4ecd-9417-02664672c644","remote_addr":"[fd00:1122:3344:101::c]:41976","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":5339,"response_code":"204"}
{"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:16.701344069Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:897"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:16.972438306Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:59320"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:16.972549487Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:17.061605406Z","hostname":"buskin","pid":1555,"uri":"/inventory","method":"GET","req_id":"470ed785-7f70-4daa-884e-9e06af74b134","remote_addr":"[fd00:1122:3344:101::c]:59320","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":89119,"response_code":"200"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:17.06199358Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Reading ledger from /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:17.062060341Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:17.062573086Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:17.062853219Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"GET","req_id":"7a8e5870-3133-45d3-adcc-183e79d791d0","remote_addr":"[fd00:1122:3344:101::c]:59320","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":951,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:17.580238086Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:62779"}
{"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:17.580470358Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:725","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2811064c-b3b2-438e-8e0c-aa9a3bfa436c, kind: Service { id: ac507aba-8f76-47c6-bd22-6dff420215b2 }, name: Name(\"external-dns-ac507aba-8f76-47c6-bd22-6dff420215b2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 139])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b39a7e8e-2dea-45da-9a7a-6977886585dc, kind: Service { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa }, name: Name(\"external-dns-b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 77])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6ab9e3b5-f385-4e7e-9bc6-6c4f7a443782, kind: Service { id: c108631e-f757-4a42-b510-28307153f426 }, name: Name(\"nexus-c108631e-f757-4a42-b510-28307153f426\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 191])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 97294434-1769-4a97-a97a-20d887e8c50a, kind: Service { id: 37d6282c-2511-4c34-b419-e68b280bef44 }, name: Name(\"nexus-37d6282c-2511-4c34-b419-e68b280bef44\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 88])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ae27ee1d-0f9e-4fca-b104-9220b85a9848, kind: Service { id: 6caaab71-9cdd-4faa-b082-987c9dd43604 }, name: Name(\"nexus-6caaab71-9cdd-4faa-b082-987c9dd43604\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 147])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:17.580531019Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:17.580733251Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:17.580748631Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:17.580762332Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[]","port":"\"opte0\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:17.580772841Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:17.580794392Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:17.580819672Z","hostname":"buskin","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"7dce154e-0237-48a4-8e4d-cb6781d481b6","remote_addr":"[fd00:1122:3344:101::a]:62779","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":501,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:17.968247335Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:63996"}
{"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:17.968511778Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:725","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2811064c-b3b2-438e-8e0c-aa9a3bfa436c, kind: Service { id: ac507aba-8f76-47c6-bd22-6dff420215b2 }, name: Name(\"external-dns-ac507aba-8f76-47c6-bd22-6dff420215b2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 139])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b39a7e8e-2dea-45da-9a7a-6977886585dc, kind: Service { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa }, name: Name(\"external-dns-b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 77])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6ab9e3b5-f385-4e7e-9bc6-6c4f7a443782, kind: Service { id: c108631e-f757-4a42-b510-28307153f426 }, name: Name(\"nexus-c108631e-f757-4a42-b510-28307153f426\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 191])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 97294434-1769-4a97-a97a-20d887e8c50a, kind: Service { id: 37d6282c-2511-4c34-b419-e68b280bef44 }, name: Name(\"nexus-37d6282c-2511-4c34-b419-e68b280bef44\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 88])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ae27ee1d-0f9e-4fca-b104-9220b85a9848, kind: Service { id: 6caaab71-9cdd-4faa-b082-987c9dd43604 }, name: Name(\"nexus-6caaab71-9cdd-4faa-b082-987c9dd43604\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 147])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:17.968596749Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:17.968607709Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:17.968617779Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:17.968625599Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[]","port":"\"opte0\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:17.968641099Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:17.96866255Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:17.9686814Z","hostname":"buskin","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"3518908a-e77f-440e-b07e-abbf897d2550","remote_addr":"[fd00:1122:3344:101::c]:63996","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":367,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:18.358354467Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:52406"}
{"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:18.35856868Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:725","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2811064c-b3b2-438e-8e0c-aa9a3bfa436c, kind: Service { id: ac507aba-8f76-47c6-bd22-6dff420215b2 }, name: Name(\"external-dns-ac507aba-8f76-47c6-bd22-6dff420215b2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 139])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b39a7e8e-2dea-45da-9a7a-6977886585dc, kind: Service { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa }, name: Name(\"external-dns-b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 77])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6ab9e3b5-f385-4e7e-9bc6-6c4f7a443782, kind: Service { id: c108631e-f757-4a42-b510-28307153f426 }, name: Name(\"nexus-c108631e-f757-4a42-b510-28307153f426\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 191])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 97294434-1769-4a97-a97a-20d887e8c50a, kind: Service { id: 37d6282c-2511-4c34-b419-e68b280bef44 }, name: Name(\"nexus-37d6282c-2511-4c34-b419-e68b280bef44\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 88])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ae27ee1d-0f9e-4fca-b104-9220b85a9848, kind: Service { id: 6caaab71-9cdd-4faa-b082-987c9dd43604 }, name: Name(\"nexus-6caaab71-9cdd-4faa-b082-987c9dd43604\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 147])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:18.35862142Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:18.35864528Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:18.358656401Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:18.358666381Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[]","port":"\"opte0\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:18.358698601Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:18.358714451Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:18.358737992Z","hostname":"buskin","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"69bacbe8-f066-43e5-91e8-57b18f2e9356","remote_addr":"[fd00:1122:3344:101::b]:52406","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":289,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:22.174108771Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:41876"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:22.175309274Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"d486a488-3492-45b5-b8c6-902bbad6e831","remote_addr":"[fd00:1122:3344:101::c]:41876","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":1169,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:22.24018509Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"50998684-9e37-4aa8-80cc-4b4ec461f546","remote_addr":"[fd00:1122:3344:101::c]:41876","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":1232,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:30.938773172Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:60465"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:30.938854763Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"66b3aa0f-9624-4ceb-870e-b0293505b8e5","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:60465","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":65,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:34.467496646Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:46118"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:34.467633677Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"67637d86-31f5-49c6-9083-26b68942a002","remote_addr":"[fd00:1122:3344:101::a]:46118","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":98,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:34.475928198Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"6e23fac0-8705-4ef5-a1ee-00d37a489c93","remote_addr":"[fd00:1122:3344:101::a]:46118","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":128,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:34.504925963Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:50542"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:34.506077335Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"8cb8ba01-64f7-438b-b5ed-859ba77cbddf","remote_addr":"[fd00:1122:3344:101::c]:50542","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":1128,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:34.513286354Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"d5181845-6938-4673-9965-1e717678e29c","remote_addr":"[fd00:1122:3344:101::c]:50542","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":66,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:34.69962151Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:38192"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:34.702135918Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"3e379438-6ffc-4d6f-9fbb-d7683a920418","remote_addr":"[fd00:1122:3344:101::c]:38192","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":2482,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:34.717161451Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:49935"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:34.717276942Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"57a72f0f-e054-4619-b38e-596c622ea073","remote_addr":"[fd00:1122:3344:101::a]:49935","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":82,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:34.819781477Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:58014"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:34.935380334Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"1c306638-fd51-414d-831d-663ad0a13f2c","remote_addr":"[fd00:1122:3344:101::c]:58014","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":115553,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:34.935538936Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:40937"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:34.93588812Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"b5c353d1-d7e7-456a-87be-0c75950b0284","remote_addr":"[fd00:1122:3344:101::b]:40937","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":318,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:34.936500996Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"f9db8766-5c9e-4a77-91b6-5678cd41f34c","remote_addr":"[fd00:1122:3344:101::c]:58014","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":801,"response_code":"200"}
{"msg":"Attempting to update network config with generation: 2, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:34.958390244Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:34.958449585Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:39614"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:34.958524096Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"2fce4265-aaef-44bb-a04a-eaa31397ac27","remote_addr":"[fd00:1122:3344:101::b]:39614","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":51,"response_code":"200"}
{"msg":"Writing ledger to /pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:34.959000261Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:34.960214694Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"}
{"msg":"Broadcasting network config with generation 2","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:34.960926152Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:34.960939362Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"PUT","req_id":"0b187c93-e822-4f03-b8d3-cc0036b89699","remote_addr":"[fd00:1122:3344:101::c]:58014","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":3757,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:34.967484553Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"87254950-1f31-4eb0-859c-e891c0091ea6","remote_addr":"[fd00:1122:3344:101::b]:39614","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":208,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:34.977821476Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:47353"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:35.040583958Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"28c8378a-bab3-4b89-81ea-15ee1c515f89","remote_addr":"[fd00:1122:3344:101::a]:47353","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":62728,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:35.040964812Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"b7e97d39-fe9e-4e8e-a912-5fad92310be4","remote_addr":"[fd00:1122:3344:101::a]:47353","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":112,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:35.355101868Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:33301"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:35.403732567Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"ad64616c-0bec-45eb-9f9d-a3eb2d85701b","remote_addr":"[fd00:1122:3344:101::b]:33301","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":48561,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-10T23:59:35.404093041Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"45c00102-c6a4-44ad-b2f2-57a7df3413ae","remote_addr":"[fd00:1122:3344:101::b]:33301","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":96,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:00.97114989Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42803"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:00.971246231Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"185fa317-6d01-4009-a05f-e0eba1397630","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42803","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":83,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:04.415282742Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:40940"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:04.415429564Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"6f0fb4d4-bd86-4b6f-83fc-536967d99afe","remote_addr":"[fd00:1122:3344:101::c]:40940","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":119,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:04.423202908Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"ce9ee25f-d737-442d-9382-f7ca1e92c985","remote_addr":"[fd00:1122:3344:101::c]:40940","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":113,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:04.438751417Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:64109"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:04.43895253Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"b2197440-a690-4cf1-b707-6eb08efbb255","remote_addr":"[fd00:1122:3344:101::c]:64109","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":146,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:04.495491154Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:45379"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:04.495647616Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"190f6973-dc57-4794-b794-94360460d5e8","remote_addr":"[fd00:1122:3344:101::a]:45379","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":113,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:04.503670494Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:61329"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:04.503713924Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"0fc1dc0f-e3fd-4a41-8053-30d4fa73d44a","remote_addr":"[fd00:1122:3344:101::a]:61329","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":25,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:04.515072318Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"e4f54289-34a2-4465-b9d6-0409949902a7","remote_addr":"[fd00:1122:3344:101::a]:61329","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":75,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:04.578678559Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:60306"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:04.630196739Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"1f040010-2b6b-40e4-aae8-7a648a7e3172","remote_addr":"[fd00:1122:3344:101::c]:60306","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":51488,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:04.630491603Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"aadc9da1-f2a7-457f-9a4a-abde9386bb41","remote_addr":"[fd00:1122:3344:101::c]:60306","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":109,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:04.651724573Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:58530"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:04.699271941Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"7ab2e7a2-742f-4465-a6d3-68e28255f906","remote_addr":"[fd00:1122:3344:101::a]:58530","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":47452,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:04.699510573Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"8d48c727-67ea-41d3-a6f0-0fdbd0ab713f","remote_addr":"[fd00:1122:3344:101::a]:58530","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":91,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:04.759981851Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:59549"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:04.760897751Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"5c93f9b8-2f7a-425c-87ec-112037e6c2c1","remote_addr":"[fd00:1122:3344:101::b]:59549","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":898,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:04.857691193Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:48961"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:04.857781394Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"e61cdf95-0155-49cf-894d-62344f25a72f","remote_addr":"[fd00:1122:3344:101::b]:48961","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":52,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:04.865725331Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"e640a33b-d943-4a35-a7ef-d4d3b04108c3","remote_addr":"[fd00:1122:3344:101::b]:48961","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":331,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:05.01549938Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:48076"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:05.061891454Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"a21daacb-ace3-4f67-97ba-9d9b90495219","remote_addr":"[fd00:1122:3344:101::b]:48076","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":46357,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:05.062141317Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"08d701b3-f88a-481c-af78-004733b499d3","remote_addr":"[fd00:1122:3344:101::b]:48076","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":99,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:31.002675339Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:53628"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:31.00277542Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"05b0aacf-856b-4ed2-aeee-ebad7d2d8584","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:53628","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":93,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:34.470129602Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:39261"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:34.470270904Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"ad23edb0-a7fa-45cd-a4b0-cec54040dc7d","remote_addr":"[fd00:1122:3344:101::c]:39261","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":125,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:34.477416362Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"3e6322ef-c26f-42f1-a9f0-be334aa6b80b","remote_addr":"[fd00:1122:3344:101::c]:39261","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":109,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:34.536993259Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:61735"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:34.537177861Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"1ffb71fd-2782-41c6-80d8-c3df30a7c378","remote_addr":"[fd00:1122:3344:101::c]:61735","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":167,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:34.602656804Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:51073"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:34.604222101Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"ea64459f-4a8f-4557-8d2a-e7cc8408f790","remote_addr":"[fd00:1122:3344:101::a]:51073","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":1538,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:34.613473341Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"b023b189-c5e9-4a99-8cb7-951f5fd645d4","remote_addr":"[fd00:1122:3344:101::a]:51073","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":111,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:34.696651226Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:47027"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:34.696853298Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"3e057ea2-87ce-4097-8a48-4f5b0a37c093","remote_addr":"[fd00:1122:3344:101::a]:47027","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":164,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:34.841213058Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:34484"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:34.841346649Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"880a0a38-d592-4c61-89de-d9526c3af363","remote_addr":"[fd00:1122:3344:101::b]:34484","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":111,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:34.842520802Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:36264"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:34.893094351Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"cbd703ee-58f2-4104-92f0-c2c33e0e33a1","remote_addr":"[fd00:1122:3344:101::a]:36264","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":50531,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:34.893383515Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"8a935b64-cc51-416f-87a3-985878e070c1","remote_addr":"[fd00:1122:3344:101::a]:36264","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":105,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:34.918789741Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:32961"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:34.965541659Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"d71dcd1f-bd7e-4516-b299-35ea9fafd509","remote_addr":"[fd00:1122:3344:101::c]:32961","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":46665,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:34.965813192Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"fdc2b353-5c5d-45cb-9829-cd17a3bfa9ae","remote_addr":"[fd00:1122:3344:101::c]:32961","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":103,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:35.045735021Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:60902"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:35.046666932Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"625f3e77-6db6-408a-9953-4e684e05f9c5","remote_addr":"[fd00:1122:3344:101::b]:60902","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":869,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:35.054445293Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"2c1aa265-d337-49d6-8367-d2a3a0694090","remote_addr":"[fd00:1122:3344:101::b]:60902","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":129,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:35.238608559Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:32888"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:35.287125486Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"0f14ac28-6504-4be6-8b55-44abb9111d27","remote_addr":"[fd00:1122:3344:101::b]:32888","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":48466,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:00:35.287367629Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"bbef3891-3e87-4adf-9b4f-8c33821b41ba","remote_addr":"[fd00:1122:3344:101::b]:32888","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":98,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:01.034582268Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42150"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:01.03478259Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"912378f4-5d34-4c6c-bbe4-a343a8156fc5","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42150","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":149,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:04.394147348Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:62209"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:04.394429371Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"397a4560-80ac-4512-b19e-9fac738af748","remote_addr":"[fd00:1122:3344:101::a]:62209","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":258,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:04.449213286Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:42846"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:04.449321638Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"1613dfb6-db11-41cd-b16d-bd37ab74281b","remote_addr":"[fd00:1122:3344:101::c]:42846","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":50,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:04.457111252Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"95a9fa0f-f25e-4a7a-aff1-68a4188f6adc","remote_addr":"[fd00:1122:3344:101::c]:42846","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":899,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:04.475880467Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:50221"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:04.476022688Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"b3b46b3a-4f7f-4342-812b-d74290eb4622","remote_addr":"[fd00:1122:3344:101::c]:50221","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":105,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:04.492023082Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:51558"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:04.492069952Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"2dcbf2b6-3cd4-4a2e-9192-9ad4d7d0096f","remote_addr":"[fd00:1122:3344:101::a]:51558","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":26,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:04.498627084Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"ff3e455e-89b5-48d4-8d78-e0a35f465245","remote_addr":"[fd00:1122:3344:101::a]:51558","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":60,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:04.610173447Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:40240"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:04.66471415Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"8fb8c13b-673e-4bc9-a34f-610e46a21dda","remote_addr":"[fd00:1122:3344:101::a]:40240","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":54479,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:04.664866451Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:43905"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:04.668957276Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"f0830518-3357-4f1e-99da-241631e06102","remote_addr":"[fd00:1122:3344:101::a]:40240","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":3861,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:04.709382888Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"930d429f-10bf-4ee2-98cc-de0630f7beb4","remote_addr":"[fd00:1122:3344:101::c]:43905","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":44434,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:04.709755764Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"5324a28c-aff5-46ef-9007-875b69d4b352","remote_addr":"[fd00:1122:3344:101::c]:43905","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":181,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:04.731805339Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:34635"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:04.73190873Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"03d4bc9c-54ce-4503-bd15-0384be9cc423","remote_addr":"[fd00:1122:3344:101::b]:34635","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":55,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:04.740102949Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"4ba1ea3c-33bc-4c7b-8751-1709bbb57bfc","remote_addr":"[fd00:1122:3344:101::b]:34635","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":693,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:04.778032472Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:36919"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:04.778172013Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"b6303220-6fe4-44be-8838-53a7d131c1b5","remote_addr":"[fd00:1122:3344:101::b]:36919","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":113,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:04.973393666Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:49946"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:05.022160907Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"716fa694-5ff7-477d-9f3e-cb0f7dffff60","remote_addr":"[fd00:1122:3344:101::b]:49946","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":48677,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:05.02252912Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"8a2ec187-7e38-4433-be10-9360b405459d","remote_addr":"[fd00:1122:3344:101::b]:49946","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":76,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:31.067123026Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63128"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:31.068063486Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"a922614b-16a9-451a-bfbc-4cea49c8b323","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63128","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":908,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:34.381000689Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:48267"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:34.381173881Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"6a6ce585-6e3f-4349-b872-4a410ec5467c","remote_addr":"[fd00:1122:3344:101::a]:48267","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":131,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:34.389117688Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"e059ee29-71ed-4472-97fd-f51a4b904dff","remote_addr":"[fd00:1122:3344:101::a]:48267","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":916,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:34.430269975Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:65024"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:34.430415406Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"a5a030c7-83bd-4b29-9eee-2027c1c98ddb","remote_addr":"[fd00:1122:3344:101::a]:65024","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":119,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:34.568844932Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:58795"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:34.568995924Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"65633800-473c-4499-9f3c-e70e4d2009cd","remote_addr":"[fd00:1122:3344:101::c]:58795","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":100,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:34.660454008Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:56965"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:34.66058876Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"7faf6328-7d54-47c9-b14d-3ee00e6dc045","remote_addr":"[fd00:1122:3344:101::c]:56965","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":108,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:34.668856469Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"765c46ac-2fa5-47e7-ba45-a91801f1e791","remote_addr":"[fd00:1122:3344:101::c]:56965","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":115,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:34.734091949Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:58913"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:34.785915052Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"3d7683e8-719a-4384-8dcc-e4cd07ca8935","remote_addr":"[fd00:1122:3344:101::a]:58913","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":51786,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:34.786173155Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"5ac70575-50cb-450a-b56a-230a249dc906","remote_addr":"[fd00:1122:3344:101::a]:58913","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":115,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:34.884063659Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:58352"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:34.884253312Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"c51bcabe-5ee7-4575-abef-19b5642a0d1b","remote_addr":"[fd00:1122:3344:101::b]:58352","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":160,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:34.957276246Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:62161"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:35.008618124Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"e29c1851-89f7-49c9-92bc-bcdbefb9bd22","remote_addr":"[fd00:1122:3344:101::c]:62161","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":51280,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:35.008882617Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:65290"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:35.008938757Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"f377ad47-8e63-4476-a1e1-2d294b09f0d9","remote_addr":"[fd00:1122:3344:101::c]:62161","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":130,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:35.009069089Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"0ff3de52-0576-47f2-b4b8-a2cb10e0ff61","remote_addr":"[fd00:1122:3344:101::b]:65290","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":100,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:35.017927035Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"43972384-d595-4349-95a1-b0f572403dcb","remote_addr":"[fd00:1122:3344:101::b]:65290","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":834,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:35.212917995Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:50653"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:35.257901915Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"0c47b12e-d1d8-4fbd-a71b-4fbf91e41aed","remote_addr":"[fd00:1122:3344:101::b]:50653","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":44951,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:35.258162937Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"c6c4774d-7216-4ede-9938-3be0fb187bb8","remote_addr":"[fd00:1122:3344:101::b]:50653","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":104,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:35.77259772Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:63316","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","component":"producer-server","component":"MetricsManager","sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::d]:59433"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:01:35.788405682Z","hostname":"buskin","pid":1555,"uri":"/0dcbb472-7cd1-4e42-9116-fc1cc27da573","method":"GET","req_id":"a0bbe4b5-9fdd-46e6-9cbc-3d6e2310c1eb","remote_addr":"[fd00:1122:3344:101::d]:59433","local_addr":"[fd00:1122:3344:101::1]:63316","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":15038,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:01.100730237Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:35067"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:01.100839458Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"ba1d7d6b-3b81-4d38-bb68-6ee1a067b3f6","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:35067","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":88,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:04.316913989Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:34398"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:04.317088181Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"a273f60e-3335-4b63-912e-7b65845d3540","remote_addr":"[fd00:1122:3344:101::a]:34398","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":134,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:04.324021986Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"56a16122-c984-4bca-bcd6-ac1cae8c931c","remote_addr":"[fd00:1122:3344:101::a]:34398","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":125,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:04.329992221Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:43550"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:04.330133553Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"d48987ac-de62-4d75-9a34-b398ed110b21","remote_addr":"[fd00:1122:3344:101::a]:43550","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":103,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:04.485685684Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:56642"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:04.485826746Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"6c4a55cb-8077-43a3-95bc-baf29d7b3f6d","remote_addr":"[fd00:1122:3344:101::c]:56642","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":97,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:04.530291599Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:46610"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:04.585124825Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"4133f9de-5dd3-43f4-98f6-79c0dc6ad1ea","remote_addr":"[fd00:1122:3344:101::a]:46610","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":54777,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:04.585260517Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:40543"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:04.585448739Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"075cf13a-bdc6-4a4a-8737-0a3a9199ba2b","remote_addr":"[fd00:1122:3344:101::c]:40543","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":112,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:04.585613211Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"025ac31b-fe8c-4c23-855e-38fb2959b329","remote_addr":"[fd00:1122:3344:101::a]:46610","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":189,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:04.592235563Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"a8509f40-ee5f-4746-b4a6-feadb667fed9","remote_addr":"[fd00:1122:3344:101::c]:40543","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":155,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:04.658897297Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:65471"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:04.709220815Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"f5930898-7c00-46e1-94ce-1a558f18894e","remote_addr":"[fd00:1122:3344:101::c]:65471","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":50283,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:04.710276276Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"2e177191-f840-4caf-8a1c-60f0569694ff","remote_addr":"[fd00:1122:3344:101::c]:65471","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":884,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:04.756090384Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:63784"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:04.756251385Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"fdbfc331-afac-4d21-9cd7-ae6693d8a209","remote_addr":"[fd00:1122:3344:101::b]:63784","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":90,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:04.763495474Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"24dd7e1f-dcd4-45c0-a84d-f50ce793f19f","remote_addr":"[fd00:1122:3344:101::b]:63784","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":111,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:04.864704365Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:61359"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:04.865170479Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"d40bdbfd-5678-4511-8fa7-b74b6cc9b2df","remote_addr":"[fd00:1122:3344:101::b]:61359","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":432,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:04.958591675Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:37519"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:05.003594015Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"3a0e9b7c-8fe8-4997-900e-3d017bb1fd30","remote_addr":"[fd00:1122:3344:101::b]:37519","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":44947,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:05.003871038Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"52c1be5b-82d6-4cfa-ab52-5d406aabb6e5","remote_addr":"[fd00:1122:3344:101::b]:37519","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":103,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:05.773744459Z","hostname":"buskin","pid":1555,"uri":"/0dcbb472-7cd1-4e42-9116-fc1cc27da573","method":"GET","req_id":"298715f0-6527-45bf-ba89-dd6ac4fca307","remote_addr":"[fd00:1122:3344:101::d]:59433","local_addr":"[fd00:1122:3344:101::1]:63316","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":713,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:31.134365616Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:59570"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:31.134441048Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"6d6b335d-a82f-48ad-b715-a603ef85fd0d","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:59570","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":59,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:34.411332878Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:53317"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:34.41148244Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"24f0ab22-8ac8-405e-8320-bc8a3f56d7b3","remote_addr":"[fd00:1122:3344:101::a]:53317","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":97,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:34.417703637Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"0ad28128-9e38-458a-9b07-0959f1551e60","remote_addr":"[fd00:1122:3344:101::a]:53317","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":122,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:34.556408956Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:39376"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:34.556461986Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"0160b9f4-fab0-4ed9-9561-d7d7b36e6974","remote_addr":"[fd00:1122:3344:101::c]:39376","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":35,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:34.562494072Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"9528637c-bf2c-4e5c-836e-6e2e910c64b7","remote_addr":"[fd00:1122:3344:101::c]:39376","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":76,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:34.565459674Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:51293"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:34.565616996Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"5e1c006a-d64f-47ed-9f4d-4aa481fb50e1","remote_addr":"[fd00:1122:3344:101::a]:51293","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":115,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:34.703110395Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:40820"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:34.703286525Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"b7b0332e-e0b7-4f23-beab-e79e28efefe3","remote_addr":"[fd00:1122:3344:101::c]:40820","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":156,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:34.839576475Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:33312"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:34.840499045Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"915a504f-7140-498c-b3ea-8287a46624b1","remote_addr":"[fd00:1122:3344:101::b]:33312","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":855,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:34.84194341Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:48697"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:34.896514334Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"391cd28a-7bf3-477d-b458-4ceb421d9f73","remote_addr":"[fd00:1122:3344:101::a]:48697","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":54517,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:34.896923828Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:51476"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:34.897194021Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"b7c7116c-4e48-4755-a7fe-2a20a7044145","remote_addr":"[fd00:1122:3344:101::a]:48697","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":374,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:34.897265512Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"44a83e6e-ec87-42f6-b1d2-b5fca50273bd","remote_addr":"[fd00:1122:3344:101::b]:33312","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":312,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:34.948999384Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"6889eada-d3bb-4970-bc18-6262ea43e866","remote_addr":"[fd00:1122:3344:101::c]:51476","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":51936,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:34.949380758Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"f0538b9c-e72f-4ce8-9d9f-cf0c260d7f47","remote_addr":"[fd00:1122:3344:101::c]:51476","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":143,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:35.10587952Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:49226"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:35.106052322Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"b0931fbb-b120-44ee-95b8-0fecaa060bf5","remote_addr":"[fd00:1122:3344:101::b]:49226","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":147,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:35.266573288Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:34335"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:35.313619989Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"e6ad3aef-dace-4868-b34d-7d748da74973","remote_addr":"[fd00:1122:3344:101::b]:34335","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":46977,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:35.314373317Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"85156388-2269-4c0b-b637-14aa995794f2","remote_addr":"[fd00:1122:3344:101::b]:34335","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":615,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:02:35.774393468Z","hostname":"buskin","pid":1555,"uri":"/0dcbb472-7cd1-4e42-9116-fc1cc27da573","method":"GET","req_id":"03e730b6-cc6e-43f5-b0d3-03a835657fc2","remote_addr":"[fd00:1122:3344:101::d]:59433","local_addr":"[fd00:1122:3344:101::1]:63316","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":1301,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:01.168301746Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:40152"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:01.169204316Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"246e07f0-7a3a-4e1e-93d6-9b43093211fd","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:40152","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":891,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:04.331938396Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:38661"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:04.332128908Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"dafc6611-6c57-42e4-9aa9-c145dba546c7","remote_addr":"[fd00:1122:3344:101::a]:38661","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":172,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:04.428369065Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:48926"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:04.428453215Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"7319e28d-8058-41b8-81f7-9d099b9e0336","remote_addr":"[fd00:1122:3344:101::a]:48926","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":38,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:04.435728605Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"f1ee9a27-009c-40a3-8c7b-3e4fb21c2742","remote_addr":"[fd00:1122:3344:101::a]:48926","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":867,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:04.46750043Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:46711"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:04.467577201Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"7889d40e-06d4-4ea5-8c68-79f72e937da7","remote_addr":"[fd00:1122:3344:101::c]:46711","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":20,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:04.474096182Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"992d629a-9f82-4da7-9e8a-3829fe5fa538","remote_addr":"[fd00:1122:3344:101::c]:46711","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":71,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:04.532648219Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:56057"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:04.580555689Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"d8be60a3-429a-41c4-a7b7-afa80cd6a42f","remote_addr":"[fd00:1122:3344:101::a]:56057","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":47857,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:04.580936274Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"b88b9804-30be-4e9b-a88b-5c0ef9ea866e","remote_addr":"[fd00:1122:3344:101::a]:56057","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":105,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:04.639768283Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:56936"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:04.639957075Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"4342d967-9c58-4282-a23c-37d424a81959","remote_addr":"[fd00:1122:3344:101::c]:56936","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":164,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:04.738112642Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:48545"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:04.78664193Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"e4c093e3-08f1-40f2-9145-de1ef879b64c","remote_addr":"[fd00:1122:3344:101::c]:48545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":48467,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:04.786874483Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"226fb40e-390a-406c-a3db-60b5a8bd512b","remote_addr":"[fd00:1122:3344:101::c]:48545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":93,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:05.057226752Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:59297"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:05.057992711Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"55214f04-b006-4fd8-987e-033828337009","remote_addr":"[fd00:1122:3344:101::b]:59297","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":719,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:05.063920265Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:56369"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:05.064115598Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"9ea3c77d-c877-4b1e-b386-8c33208265a8","remote_addr":"[fd00:1122:3344:101::b]:56369","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":151,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:05.065762285Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"e1fd941a-9013-4d74-8447-81ebdcbd96e2","remote_addr":"[fd00:1122:3344:101::b]:59297","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":611,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:05.187668341Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:55039"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:05.233091905Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"143878a0-3c53-4e2a-a299-4d63562e2e3f","remote_addr":"[fd00:1122:3344:101::b]:55039","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":45368,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:05.233936154Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"a3801b65-ece8-4752-b88a-2b6d9e1dc961","remote_addr":"[fd00:1122:3344:101::b]:55039","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":697,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:05.774729645Z","hostname":"buskin","pid":1555,"uri":"/0dcbb472-7cd1-4e42-9116-fc1cc27da573","method":"GET","req_id":"6b1cc305-d860-4322-8de0-30fbe8effb82","remote_addr":"[fd00:1122:3344:101::d]:59433","local_addr":"[fd00:1122:3344:101::1]:63316","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":798,"response_code":"200"}
{"msg":"running automatic periodic zone bundle cleanup","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:17.608088354Z","hostname":"buskin","pid":1555,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:300"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:17.608169205Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:17.645850025Z","hostname":"buskin","pid":1555,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:289","time_to_next_cleanup":"599.99999801s","next_cleanup":"Instant { tv_sec: 2221, tv_nsec: 6848505 }"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:31.202467922Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:46486"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:31.202563483Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"62a9ee85-c47a-4861-9798-a22ab90541c4","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:46486","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":65,"response_code":"200"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:31.446876079Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:34.396827596Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:41454"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:34.397044948Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"34101e7e-a639-4cf9-9e55-2afbbf87fd00","remote_addr":"[fd00:1122:3344:101::a]:41454","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":182,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:34.582207942Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:56619"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:34.582350063Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"80662d97-5081-4df6-b99b-7b61a9f5da24","remote_addr":"[fd00:1122:3344:101::c]:56619","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":118,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:34.590502662Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:59355"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:34.590586653Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"87461883-3a44-48a1-8d83-c92b0ce6027e","remote_addr":"[fd00:1122:3344:101::c]:59355","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":47,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:34.597822901Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"f6a7fcf7-d6b2-430e-af22-e5da209ef5e8","remote_addr":"[fd00:1122:3344:101::c]:59355","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":91,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:34.633814383Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:48972"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:34.633890434Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"989185d8-6011-4fe0-8d14-a1c4ce996f6d","remote_addr":"[fd00:1122:3344:101::a]:48972","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":55,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:34.641457836Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"1fc00d26-9d7d-4370-8d40-d4a6e402d293","remote_addr":"[fd00:1122:3344:101::a]:48972","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":114,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:34.694784276Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:48020"}
{"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:34.694941837Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:725","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2811064c-b3b2-438e-8e0c-aa9a3bfa436c, kind: Service { id: ac507aba-8f76-47c6-bd22-6dff420215b2 }, name: Name(\"external-dns-ac507aba-8f76-47c6-bd22-6dff420215b2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 139])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b39a7e8e-2dea-45da-9a7a-6977886585dc, kind: Service { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa }, name: Name(\"external-dns-b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 77])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6ab9e3b5-f385-4e7e-9bc6-6c4f7a443782, kind: Service { id: c108631e-f757-4a42-b510-28307153f426 }, name: Name(\"nexus-c108631e-f757-4a42-b510-28307153f426\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 191])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 97294434-1769-4a97-a97a-20d887e8c50a, kind: Service { id: 37d6282c-2511-4c34-b419-e68b280bef44 }, name: Name(\"nexus-37d6282c-2511-4c34-b419-e68b280bef44\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 88])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ae27ee1d-0f9e-4fca-b104-9220b85a9848, kind: Service { id: 6caaab71-9cdd-4faa-b082-987c9dd43604 }, name: Name(\"nexus-6caaab71-9cdd-4faa-b082-987c9dd43604\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 147])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:34.694982798Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:34.695009158Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:34.695017338Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:34.695024889Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[]","port":"\"opte0\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:34.695039729Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:34.695052949Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:34.695074389Z","hostname":"buskin","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"185c009d-6238-465a-95ac-e2d927195b99","remote_addr":"[fd00:1122:3344:101::a]:48020","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":264,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:34.696342583Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:39598"}
{"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:34.696517595Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:725","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2811064c-b3b2-438e-8e0c-aa9a3bfa436c, kind: Service { id: ac507aba-8f76-47c6-bd22-6dff420215b2 }, name: Name(\"external-dns-ac507aba-8f76-47c6-bd22-6dff420215b2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 139])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b39a7e8e-2dea-45da-9a7a-6977886585dc, kind: Service { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa }, name: Name(\"external-dns-b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 77])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6ab9e3b5-f385-4e7e-9bc6-6c4f7a443782, kind: Service { id: c108631e-f757-4a42-b510-28307153f426 }, name: Name(\"nexus-c108631e-f757-4a42-b510-28307153f426\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 191])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 97294434-1769-4a97-a97a-20d887e8c50a, kind: Service { id: 37d6282c-2511-4c34-b419-e68b280bef44 }, name: Name(\"nexus-37d6282c-2511-4c34-b419-e68b280bef44\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 88])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ae27ee1d-0f9e-4fca-b104-9220b85a9848, kind: Service { id: 6caaab71-9cdd-4faa-b082-987c9dd43604 }, name: Name(\"nexus-6caaab71-9cdd-4faa-b082-987c9dd43604\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 147])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:34.696551055Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:34.696557435Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:34.696566405Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:34.696570585Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[]","port":"\"opte0\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:34.696580855Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:34.696596306Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:34.696609286Z","hostname":"buskin","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"ecc5c6f2-46f0-499e-b6cb-d938d87ed993","remote_addr":"[fd00:1122:3344:101::c]:39598","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":186,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:34.830334569Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:48634"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:34.878638474Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"edb786ee-d2ff-49b0-96ac-dbe0c1070b4f","remote_addr":"[fd00:1122:3344:101::a]:48634","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":48256,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:34.878914957Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"fd124024-3b92-4e84-ad4d-813c647ef630","remote_addr":"[fd00:1122:3344:101::a]:48634","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":120,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:34.913039338Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:64429"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:34.963706729Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"ff85ee3a-4967-4d43-9a39-c6bdf1a337fd","remote_addr":"[fd00:1122:3344:101::c]:64429","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":50575,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:34.964022163Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"12da9db6-761e-4172-baf1-ce07e4e50110","remote_addr":"[fd00:1122:3344:101::c]:64429","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":113,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:35.064124011Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:50432"}
{"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:35.064423234Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:725","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2811064c-b3b2-438e-8e0c-aa9a3bfa436c, kind: Service { id: ac507aba-8f76-47c6-bd22-6dff420215b2 }, name: Name(\"external-dns-ac507aba-8f76-47c6-bd22-6dff420215b2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 139])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b39a7e8e-2dea-45da-9a7a-6977886585dc, kind: Service { id: b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa }, name: Name(\"external-dns-b2ff1104-4a6c-4254-8ce9-75a1ec34f1fa\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 77])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6ab9e3b5-f385-4e7e-9bc6-6c4f7a443782, kind: Service { id: c108631e-f757-4a42-b510-28307153f426 }, name: Name(\"nexus-c108631e-f757-4a42-b510-28307153f426\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 191])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 97294434-1769-4a97-a97a-20d887e8c50a, kind: Service { id: 37d6282c-2511-4c34-b419-e68b280bef44 }, name: Name(\"nexus-37d6282c-2511-4c34-b419-e68b280bef44\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 230, 88])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ae27ee1d-0f9e-4fca-b104-9220b85a9848, kind: Service { id: 6caaab71-9cdd-4faa-b082-987c9dd43604 }, name: Name(\"nexus-6caaab71-9cdd-4faa-b082-987c9dd43604\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 147])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:35.064474195Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:35.064516135Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:35.064535046Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:35.064545266Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[]","port":"\"opte0\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:35.064562776Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:35.064576236Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:35.064593166Z","hostname":"buskin","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"adc1569b-0b12-4cd0-b877-bc094a48a4b0","remote_addr":"[fd00:1122:3344:101::b]:50432","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":393,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:35.114841753Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:48716"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:35.115008664Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"e1babe5c-6143-4d62-8b8f-0629373fe722","remote_addr":"[fd00:1122:3344:101::b]:48716","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":120,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:35.172872144Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:36836"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:35.173857104Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"8c6f484e-f8d0-4294-b901-ae8081513f9e","remote_addr":"[fd00:1122:3344:101::b]:36836","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":909,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:35.182103654Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"89029f28-18a4-4ca3-9f6f-154d8fa19b9e","remote_addr":"[fd00:1122:3344:101::b]:36836","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":90,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:35.386757559Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:63838"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:35.43277011Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"c09295a0-eb19-45b8-a889-1f059a8ea8c0","remote_addr":"[fd00:1122:3344:101::b]:63838","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":45954,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:35.433049163Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"d3c80250-56d7-4285-b41f-89ff7a6c7d67","remote_addr":"[fd00:1122:3344:101::b]:63838","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":127,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:03:35.784580615Z","hostname":"buskin","pid":1555,"uri":"/0dcbb472-7cd1-4e42-9116-fc1cc27da573","method":"GET","req_id":"4d0d4bbc-7d61-4a8c-b12d-42a8d4006990","remote_addr":"[fd00:1122:3344:101::d]:59433","local_addr":"[fd00:1122:3344:101::1]:63316","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":421,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:01.234429653Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:59218"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:01.234513844Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"eeec5130-5b13-4fa4-bf15-b3bba09c0218","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:59218","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":70,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:04.336777655Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:41885"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:04.337828986Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"a6b9dae2-1a03-4f69-900e-f533d2d3a8f2","remote_addr":"[fd00:1122:3344:101::a]:41885","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":1021,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:04.415857615Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:65001"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:04.415950796Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"7ccaa52f-facc-4157-b28b-525f67f0bd84","remote_addr":"[fd00:1122:3344:101::a]:65001","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":70,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:04.423113924Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"280d48c1-2d4a-4631-9c60-14b33a1877dd","remote_addr":"[fd00:1122:3344:101::a]:65001","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":307,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:04.423265725Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:47316"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:04.423325716Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"670075cc-4fe0-4b2b-b832-a2bf9b4dd548","remote_addr":"[fd00:1122:3344:101::c]:47316","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":35,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:04.429301411Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"421b3e4f-38d4-4357-ba24-548062efbd8a","remote_addr":"[fd00:1122:3344:101::c]:47316","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":65,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:04.460051475Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:54552"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:04.460171107Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"694dc3d2-136a-4631-ba24-213d96eab745","remote_addr":"[fd00:1122:3344:101::c]:54552","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":92,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:04.556866198Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:48112"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:04.603280103Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"f2e5851b-93d9-4984-9311-4876ecd73b8e","remote_addr":"[fd00:1122:3344:101::a]:48112","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":46361,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:04.603415034Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:37053"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:04.607872823Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"986a89df-7dee-4c89-a9e1-31d5ba8490d6","remote_addr":"[fd00:1122:3344:101::a]:48112","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":4229,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:04.658108979Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"22fd734f-4906-4bee-a571-223c687238ab","remote_addr":"[fd00:1122:3344:101::c]:37053","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":54592,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:04.658517954Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"e15d970b-6974-4639-a84e-c74ac43db76e","remote_addr":"[fd00:1122:3344:101::c]:37053","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":142,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:04.759656743Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:52201"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:04.759765224Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"d6484adc-8c3f-4bb1-bf1d-a8899f3b083c","remote_addr":"[fd00:1122:3344:101::b]:52201","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":60,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:04.766164004Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"73a1eb29-87df-43be-a888-df9654895fe7","remote_addr":"[fd00:1122:3344:101::b]:52201","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":105,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:04.775548666Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:51939"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:04.775695457Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"5feac785-639a-4de4-b0cd-e38a81fc26c0","remote_addr":"[fd00:1122:3344:101::b]:51939","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":108,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:05.048208021Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:38261"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:05.09689848Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"06b1f66a-3c61-4b08-9b03-ba9a3e353f6c","remote_addr":"[fd00:1122:3344:101::b]:38261","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":48636,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:05.097225794Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"c4de8154-3fdd-40ea-acea-eec4459f6242","remote_addr":"[fd00:1122:3344:101::b]:38261","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":135,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:05.822732642Z","hostname":"buskin","pid":1555,"uri":"/0dcbb472-7cd1-4e42-9116-fc1cc27da573","method":"GET","req_id":"5f149156-5f6a-440b-b14e-ebdd88ae89fe","remote_addr":"[fd00:1122:3344:101::d]:59433","local_addr":"[fd00:1122:3344:101::1]:63316","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":679,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:31.266768962Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42024"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:31.266843753Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"10ca4013-1a50-4e93-926f-d284fd0b48ef","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42024","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":59,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:34.403927714Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:55467"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:34.404117976Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"b18496fc-bd3e-45aa-9256-f7777acf09a7","remote_addr":"[fd00:1122:3344:101::a]:55467","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":162,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:34.567760935Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:44133"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:34.567872367Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"d2274e87-cc24-4ccd-94b0-abbc0ca54f34","remote_addr":"[fd00:1122:3344:101::a]:44133","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":84,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:34.575177736Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"68679d3a-72a0-4ca5-a231-990e2c224ba9","remote_addr":"[fd00:1122:3344:101::a]:44133","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":108,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:34.654744581Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:41443"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:34.654940713Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"a8b191d0-0dad-4fdf-96de-3ebcde7954e0","remote_addr":"[fd00:1122:3344:101::c]:41443","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":165,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:34.671777286Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:56690"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:34.672855778Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"d877b2d5-3d40-4200-895b-1f1f87aaac2f","remote_addr":"[fd00:1122:3344:101::c]:56690","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":1019,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:34.680241798Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"b4fa282e-bc90-4c54-8ebb-13131402e3d0","remote_addr":"[fd00:1122:3344:101::c]:56690","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":87,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:34.713803103Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:57310"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:34.767933132Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"952bf5f3-32c6-4282-a41d-3843f2988b9a","remote_addr":"[fd00:1122:3344:101::a]:57310","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":54092,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:34.768936902Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"6cbd3e7a-9df7-4783-b1f3-35108b5177a8","remote_addr":"[fd00:1122:3344:101::a]:57310","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":826,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:34.802810081Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:39631"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:34.802960932Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"0d376a00-e668-4b28-9303-84ccdbe10520","remote_addr":"[fd00:1122:3344:101::b]:39631","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":88,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:34.809589655Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"4370ef87-fa0d-44af-9695-cd6516a1a035","remote_addr":"[fd00:1122:3344:101::b]:39631","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":120,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:34.819205489Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:54045"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:34.864915036Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"48ca83c2-6cf9-4a53-8287-b07df5bb080b","remote_addr":"[fd00:1122:3344:101::c]:54045","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":45655,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:34.865196889Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"f82152dd-0ef6-43c3-b7cb-891c2641b94e","remote_addr":"[fd00:1122:3344:101::c]:54045","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":105,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:35.123532908Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:48039"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:35.12373406Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"4cf57e19-4512-4c75-a5e5-9ac257e9aebc","remote_addr":"[fd00:1122:3344:101::b]:48039","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":160,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:35.30394205Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:47984"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:35.353010814Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"9849ee45-00aa-4287-94f0-6f80ef59a69b","remote_addr":"[fd00:1122:3344:101::b]:47984","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":49003,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:35.353260466Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"16074fe3-4a91-4a18-a1ad-c6bd46dcafb1","remote_addr":"[fd00:1122:3344:101::b]:47984","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":94,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:35.822446348Z","hostname":"buskin","pid":1555,"uri":"/0dcbb472-7cd1-4e42-9116-fc1cc27da573","method":"GET","req_id":"faa40c3d-c67c-4a7d-b7a3-63ebd1149037","remote_addr":"[fd00:1122:3344:101::d]:59433","local_addr":"[fd00:1122:3344:101::1]:63316","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":586,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:55.389393978Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:41504"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:55.38961552Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"bbb935c3-afe4-4f22-b2bf-9bf3bdff1117","remote_addr":"[fd00:1122:3344:101::a]:41504","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":195,"response_code":"200"}
{"msg":"Mapping virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:55.391790144Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:832","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 244, 51, 20])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(15030357) }"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:55.392900736Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"PUT","req_id":"70d3f161-6c54-4173-9b65-fe6502fc6b79","remote_addr":"[fd00:1122:3344:101::a]:41504","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":3104,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:55.5247865Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:49224"}
{"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:55.536046412Z","hostname":"buskin","pid":1555,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:548","metadata":"InstanceMetadata { silo_id: a55efdcd-0290-4b81-8d2d-5daf908e0c4c, project_id: c98e2620-3461-4f22-9e07-df9f0b69cc77 }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-07-11T00:04:55.121644Z }","instance_runtime":"InstanceRuntimeState { propolis_id: Some(351ac2ad-f19e-403f-9072-cb6eab26571f (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-07-11T00:04:55.176868087Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 6cc48e3c-3f32-4782-8b79-6756a9c586ad, kind: Instance { id: e15256d7-8018-4d03-a55a-7fe4d0ce46d6 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 244, 51, 20])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(15030357), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6cc48e3c-3f32-4782-8b79-6756a9c586ad, kind: Instance { id: e15256d7-8018-4d03-a55a-7fe4d0ce46d6 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 244, 51, 20])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(15030357), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6cc48e3c-3f32-4782-8b79-6756a9c586ad, kind: Instance { id: e15256d7-8018-4d03-a55a-7fe4d0ce46d6 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 244, 51, 20])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(15030357), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(15030357))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6cc48e3c-3f32-4782-8b79-6756a9c586ad, kind: Instance { id: e15256d7-8018-4d03-a55a-7fe4d0ce46d6 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 244, 51, 20])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(15030357), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-6a292fd7811\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 4d1164e3-12ee-4099-bf26-57101777c602, read_only_parent: Some(Volume { block_size: 512, id: 71ed95cf-a738-474a-8b14-022fa1a8ad3c, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: ea5b5e7c-bb04-4e40-8c52-f3a8a5771723, key: Some(\"+ss7k877Pv0Cf44Zn2ycdzYsvSq7D5SrNMYCvjLCNnU=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::15]:19002\", \"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::19]:19002\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 4d1164e3-12ee-4099-bf26-57101777c602, key: Some(\"SZ+QgO0myUEooY0oJyqFYqCKxlXE9jzm1hV73gU4oE4=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::15]:19003\", \"[fd00:1122:3344:101::19]:19003\", \"[fd00:1122:3344:101::13]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAACUmwDrWOtYAACbAOtYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAlJsA61jrWAAAmwDrWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiJlMTUyNTZkNy04MDE4LTRkMDMtYTU1YS03ZmU0ZDBjZTQ2ZDYiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJSzVyeVREMnlXaWRlenpsb01Na3k0OEtuaVNnQ1NKMFBFUnN0MnZPTW5TTiJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }","propolis_id":"351ac2ad-f19e-403f-9072-cb6eab26571f","instance_id":"e15256d7-8018-4d03-a55a-7fe4d0ce46d6"}
{"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:55.536152174Z","hostname":"buskin","pid":1555,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:582","instance_id":"e15256d7-8018-4d03-a55a-7fe4d0ce46d6 (instance)"}
{"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:55.536276045Z","hostname":"buskin","pid":1555,"component":"InstanceManager","file":"sled-agent/src/instance.rs:942","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 6cc48e3c-3f32-4782-8b79-6756a9c586ad, kind: Instance { id: e15256d7-8018-4d03-a55a-7fe4d0ce46d6 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 244, 51, 20])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(15030357), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6cc48e3c-3f32-4782-8b79-6756a9c586ad, kind: Instance { id: e15256d7-8018-4d03-a55a-7fe4d0ce46d6 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 244, 51, 20])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(15030357), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6cc48e3c-3f32-4782-8b79-6756a9c586ad, kind: Instance { id: e15256d7-8018-4d03-a55a-7fe4d0ce46d6 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 244, 51, 20])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(15030357), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(15030357))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6cc48e3c-3f32-4782-8b79-6756a9c586ad, kind: Instance { id: e15256d7-8018-4d03-a55a-7fe4d0ce46d6 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 244, 51, 20])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(15030357), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-6a292fd7811\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 4d1164e3-12ee-4099-bf26-57101777c602, read_only_parent: Some(Volume { block_size: 512, id: 71ed95cf-a738-474a-8b14-022fa1a8ad3c, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: ea5b5e7c-bb04-4e40-8c52-f3a8a5771723, key: Some(\"+ss7k877Pv0Cf44Zn2ycdzYsvSq7D5SrNMYCvjLCNnU=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::15]:19002\", \"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::19]:19002\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 4d1164e3-12ee-4099-bf26-57101777c602, key: Some(\"SZ+QgO0myUEooY0oJyqFYqCKxlXE9jzm1hV73gU4oE4=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::15]:19003\", \"[fd00:1122:3344:101::19]:19003\", \"[fd00:1122:3344:101::13]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAACUmwDrWOtYAACbAOtYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAlJsA61jrWAAAmwDrWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiJlMTUyNTZkNy04MDE4LTRkMDMtYTU1YS03ZmU0ZDBjZTQ2ZDYiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJSzVyeVREMnlXaWRlenpsb01Na3k0OEtuaVNnQ1NKMFBFUnN0MnZPTW5TTiJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }, instance_runtime: InstanceRuntimeState { propolis_id: Some(351ac2ad-f19e-403f-9072-cb6eab26571f (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-07-11T00:04:55.176868087Z }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-07-11T00:04:55.121644Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400 }","propolis_id":"351ac2ad-f19e-403f-9072-cb6eab26571f","instance_id":"e15256d7-8018-4d03-a55a-7fe4d0ce46d6"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:55.536788101Z","hostname":"buskin","pid":1555,"uri":"/instances/e15256d7-8018-4d03-a55a-7fe4d0ce46d6","method":"PUT","req_id":"794ba29d-0549-480b-865f-8f9cfeb05153","remote_addr":"[fd00:1122:3344:101::a]:49224","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":11950,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:55.654504671Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:53136"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:55.654900665Z","hostname":"buskin","pid":1555,"uri":"/instances/e15256d7-8018-4d03-a55a-7fe4d0ce46d6/state","method":"PUT","req_id":"d34c0d66-aa17-4b61-9bdb-5d1c5ed4f723","remote_addr":"[fd00:1122:3344:101::a]:53136","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":311,"response_code":"200"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:55.676051035Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:478","port":"Port { inner: PortInner(PortData { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 244, 51, 20]), slot: 0, vni: Vni { inner: 15030357 }, subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 }, vnic: \"vopte6\" }) }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:55.676723942Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Configuring new Omicron zone: oxz_propolis-server_351ac2ad-f19e-403f-9072-cb6eab26571f","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:55.709987714Z","hostname":"buskin","pid":1555,"instance_id":"e15256d7-8018-4d03-a55a-7fe4d0ce46d6","component":"InstanceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_propolis-server_351ac2ad-f19e-403f-9072-cb6eab26571f","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:55.731670979Z","hostname":"buskin","pid":1555,"instance_id":"e15256d7-8018-4d03-a55a-7fe4d0ce46d6","component":"InstanceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Profile for oxz_propolis-server_351ac2ad-f19e-403f-9072-cb6eab26571f:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"system/illumos/propolis-server\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlInstance0\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::1:0\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"12400\"/>\n        <propval type=\"astring\" name=\"metric_addr\" value=\"[fd00:1122:3344:101::a]:12221\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:57.349713163Z","hostname":"buskin","pid":1555,"instance_id":"e15256d7-8018-4d03-a55a-7fe4d0ce46d6","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:04:57.349950946Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_351ac2ad-f19e-403f-9072-cb6eab26571f","instance_id":"e15256d7-8018-4d03-a55a-7fe4d0ce46d6","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_propolis-server_351ac2ad-f19e-403f-9072-cb6eab26571f\") failed: Property not found. retry in 60.024913ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-11T00:04:57.517492407Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_351ac2ad-f19e-403f-9072-cb6eab26571f","instance_id":"e15256d7-8018-4d03-a55a-7fe4d0ce46d6","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_351ac2ad-f19e-403f-9072-cb6eab26571f\") failed: Property not found. retry in 74.710849ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-11T00:04:57.590254099Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_351ac2ad-f19e-403f-9072-cb6eab26571f","instance_id":"e15256d7-8018-4d03-a55a-7fe4d0ce46d6","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_351ac2ad-f19e-403f-9072-cb6eab26571f\") failed: Property not found. retry in 213.061344ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-11T00:04:57.677628089Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_351ac2ad-f19e-403f-9072-cb6eab26571f","instance_id":"e15256d7-8018-4d03-a55a-7fe4d0ce46d6","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_351ac2ad-f19e-403f-9072-cb6eab26571f\") failed: Property not found. retry in 483.562327ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-11T00:04:57.906681918Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_351ac2ad-f19e-403f-9072-cb6eab26571f","instance_id":"e15256d7-8018-4d03-a55a-7fe4d0ce46d6","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_351ac2ad-f19e-403f-9072-cb6eab26571f\") failed: Property not found. retry in 602.294394ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-11T00:04:58.406111859Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_351ac2ad-f19e-403f-9072-cb6eab26571f","instance_id":"e15256d7-8018-4d03-a55a-7fe4d0ce46d6","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_351ac2ad-f19e-403f-9072-cb6eab26571f\") failed: Property not found. retry in 728.17428ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-11T00:04:59.02337704Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_351ac2ad-f19e-403f-9072-cb6eab26571f","instance_id":"e15256d7-8018-4d03-a55a-7fe4d0ce46d6","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_351ac2ad-f19e-403f-9072-cb6eab26571f\") failed: Property not found. retry in 1.238488012s","v":0,"name":"SledAgent","level":40,"time":"2024-07-11T00:04:59.763041403Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_351ac2ad-f19e-403f-9072-cb6eab26571f","instance_id":"e15256d7-8018-4d03-a55a-7fe4d0ce46d6","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_351ac2ad-f19e-403f-9072-cb6eab26571f\") failed: Property not found. retry in 939.669856ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-11T00:05:01.012782392Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_351ac2ad-f19e-403f-9072-cb6eab26571f","instance_id":"e15256d7-8018-4d03-a55a-7fe4d0ce46d6","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:01.300035655Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:48441"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:01.300148996Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"c4a15ac8-49ea-4b7c-9ef0-cc5047ed712e","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:48441","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":87,"response_code":"200"}
{"msg":"wait for service Some(\"oxz_propolis-server_351ac2ad-f19e-403f-9072-cb6eab26571f\") failed: Property not found. retry in 566.86306ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-11T00:05:01.964240877Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_351ac2ad-f19e-403f-9072-cb6eab26571f","instance_id":"e15256d7-8018-4d03-a55a-7fe4d0ce46d6","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Started propolis in zone: oxz_propolis-server_351ac2ad-f19e-403f-9072-cb6eab26571f","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:02.561248169Z","hostname":"buskin","pid":1555,"instance_id":"e15256d7-8018-4d03-a55a-7fe4d0ce46d6","component":"InstanceManager","file":"sled-agent/src/instance.rs:1428"}
{"msg":"wait for service Some(\"oxz_propolis-server_351ac2ad-f19e-403f-9072-cb6eab26571f\") failed: Property not found. retry in 32.775996ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-11T00:05:02.57056368Z","hostname":"buskin","pid":1555,"instance_id":"e15256d7-8018-4d03-a55a-7fe4d0ce46d6","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_351ac2ad-f19e-403f-9072-cb6eab26571f\") failed: Property not found. retry in 51.802824ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-11T00:05:02.613479477Z","hostname":"buskin","pid":1555,"instance_id":"e15256d7-8018-4d03-a55a-7fe4d0ce46d6","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_351ac2ad-f19e-403f-9072-cb6eab26571f\") failed: Property not found. retry in 294.651354ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-11T00:05:02.674811824Z","hostname":"buskin","pid":1555,"instance_id":"e15256d7-8018-4d03-a55a-7fe4d0ce46d6","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:02.980506977Z","hostname":"buskin","pid":1555,"instance_id":"e15256d7-8018-4d03-a55a-7fe4d0ce46d6","component":"InstanceManager","file":"sled-agent/src/instance.rs:1437"}
{"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:03.021369461Z","hostname":"buskin","pid":1555,"instance_id":"e15256d7-8018-4d03-a55a-7fe4d0ce46d6","component":"InstanceManager","file":"sled-agent/src/instance.rs:1451"}
{"msg":"Sending ensure request to propolis: InstanceEnsureRequest { cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAACUmwDrWOtYAACbAOtYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAlJsA61jrWAAAmwDrWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiJlMTUyNTZkNy04MDE4LTRkMDMtYTU1YS03ZmU0ZDBjZTQ2ZDYiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJSzVyeVREMnlXaWRlenpsb01Na3k0OEtuaVNnQ1NKMFBFUnN0MnZPTW5TTiJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"), disks: [DiskRequest { device: \"nvme\", name: \"disk-6a292fd7811\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 4d1164e3-12ee-4099-bf26-57101777c602, read_only_parent: Some(Volume { block_size: 512, id: 71ed95cf-a738-474a-8b14-022fa1a8ad3c, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: ea5b5e7c-bb04-4e40-8c52-f3a8a5771723, key: Some(\"+ss7k877Pv0Cf44Zn2ycdzYsvSq7D5SrNMYCvjLCNnU=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::15]:19002\", \"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::19]:19002\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 4d1164e3-12ee-4099-bf26-57101777c602, key: Some(\"SZ+QgO0myUEooY0oJyqFYqCKxlXE9jzm1hV73gU4oE4=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::15]:19003\", \"[fd00:1122:3344:101::19]:19003\", \"[fd00:1122:3344:101::13]:19000\"] } }] } }], migrate: None, nics: [NetworkInterfaceRequest { name: \"vopte6\", slot: Slot(0) }], properties: InstanceProperties { bootrom_id: 00000000-0000-0000-0000-000000000000, description: \"Test description\", id: e15256d7-8018-4d03-a55a-7fe4d0ce46d6, image_id: 00000000-0000-0000-0000-000000000000, memory: 1024, metadata: InstanceMetadata { project_id: c98e2620-3461-4f22-9e07-df9f0b69cc77, silo_id: a55efdcd-0290-4b81-8d2d-5daf908e0c4c }, name: \"localshark\", vcpus: 2 } }","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:03.022515504Z","hostname":"buskin","pid":1555,"instance_id":"e15256d7-8018-4d03-a55a-7fe4d0ce46d6","component":"InstanceManager","file":"sled-agent/src/instance.rs:689"}
{"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:03.196141072Z","hostname":"buskin","pid":1555,"instance_id":"e15256d7-8018-4d03-a55a-7fe4d0ce46d6","component":"InstanceManager","file":"sled-agent/src/instance.rs:691"}
{"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Starting), migration_status: NoMigration, time: 2024-07-11T00:05:03.196779289Z }","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:03.196805379Z","hostname":"buskin","pid":1555,"instance_id":"e15256d7-8018-4d03-a55a-7fe4d0ce46d6","component":"InstanceManager","file":"sled-agent/src/instance.rs:573"}
{"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:03.196966821Z","hostname":"buskin","pid":1555,"instance_id":"e15256d7-8018-4d03-a55a-7fe4d0ce46d6","component":"InstanceManager","file":"sled-agent/src/instance.rs:595","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-07-11T00:05:03.196779289Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(351ac2ad-f19e-403f-9072-cb6eab26571f (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-07-11T00:04:55.176868087Z }","propolis_id":"351ac2ad-f19e-403f-9072-cb6eab26571f"}
{"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:03.196999791Z","hostname":"buskin","pid":1555,"instance_id":"e15256d7-8018-4d03-a55a-7fe4d0ce46d6","component":"InstanceManager","file":"sled-agent/src/instance.rs:492","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(351ac2ad-f19e-403f-9072-cb6eab26571f (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-07-11T00:04:55.176868087Z }, propolis_id: 351ac2ad-f19e-403f-9072-cb6eab26571f (propolis), vmm_state: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-07-11T00:05:03.196779289Z }, migration_state: None }","instance_id":"e15256d7-8018-4d03-a55a-7fe4d0ce46d6"}
{"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_status: NoMigration, time: 2024-07-11T00:05:03.331804257Z }","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:03.331831297Z","hostname":"buskin","pid":1555,"instance_id":"e15256d7-8018-4d03-a55a-7fe4d0ce46d6","component":"InstanceManager","file":"sled-agent/src/instance.rs:573"}
{"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:03.331851808Z","hostname":"buskin","pid":1555,"instance_id":"e15256d7-8018-4d03-a55a-7fe4d0ce46d6","component":"InstanceManager","file":"sled-agent/src/instance.rs:595","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-07-11T00:05:03.331804257Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(351ac2ad-f19e-403f-9072-cb6eab26571f (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-07-11T00:04:55.176868087Z }","propolis_id":"351ac2ad-f19e-403f-9072-cb6eab26571f"}
{"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:03.331868308Z","hostname":"buskin","pid":1555,"instance_id":"e15256d7-8018-4d03-a55a-7fe4d0ce46d6","component":"InstanceManager","file":"sled-agent/src/instance.rs:492","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(351ac2ad-f19e-403f-9072-cb6eab26571f (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-07-11T00:04:55.176868087Z }, propolis_id: 351ac2ad-f19e-403f-9072-cb6eab26571f (propolis), vmm_state: VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-07-11T00:05:03.331804257Z }, migration_state: None }","instance_id":"e15256d7-8018-4d03-a55a-7fe4d0ce46d6"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:03.368607227Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:37764"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:03.368777124Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"2f9ba266-b6d7-4332-bf04-5f30fc412534","remote_addr":"[fd00:1122:3344:101::b]:37764","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":96,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:03.417355918Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"5a6582c1-00b1-45de-9d0f-8eafda3939e3","remote_addr":"[fd00:1122:3344:101::b]:37764","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":273,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:03.456066939Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:35967"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:03.456135889Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"f61ef9f2-f67c-4910-99a4-edda9f1be140","remote_addr":"[fd00:1122:3344:101::b]:35967","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":30,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:03.4672354Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"80036d35-0d0e-49ce-8113-eb5e78a223af","remote_addr":"[fd00:1122:3344:101::b]:35967","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":821,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:04.37320297Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:43375"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:04.373417462Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"f1c2c604-d7a1-414e-ac0b-73af4a5f44ad","remote_addr":"[fd00:1122:3344:101::a]:43375","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":198,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:04.438787933Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:47219"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:04.438950705Z","hostname":"buskin","pid":1555,"uri":"/instances/e15256d7-8018-4d03-a55a-7fe4d0ce46d6/state","method":"GET","req_id":"efcb3b6b-cec7-4567-ad64-86be02b0ab27","remote_addr":"[fd00:1122:3344:101::c]:47219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":127,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:04.446092743Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:41384"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:04.446293825Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"56cf9f9c-7c55-41b9-ae46-c14f1825017d","remote_addr":"[fd00:1122:3344:101::c]:41384","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":97,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:04.462327339Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"0a25f35e-17b5-4c9a-81f2-e61dbcf58dbc","remote_addr":"[fd00:1122:3344:101::c]:41384","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":986,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:04.476369172Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:42459"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:04.476461363Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"45d3e8e6-0354-4360-8cc0-59f0193644aa","remote_addr":"[fd00:1122:3344:101::a]:42459","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":26,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:04.487132149Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:50971"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:04.488065099Z","hostname":"buskin","pid":1555,"uri":"/instances/e15256d7-8018-4d03-a55a-7fe4d0ce46d6/state","method":"GET","req_id":"0d5ba2b4-2396-427a-844d-c0c88df3ca62","remote_addr":"[fd00:1122:3344:101::a]:50971","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":889,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:04.490209983Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"fd64a4c1-e5eb-437d-8179-542d7162103d","remote_addr":"[fd00:1122:3344:101::a]:42459","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":147,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:04.498059648Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:43646"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:04.498184439Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"8545d925-22e5-4591-8f2a-fc6c6105ba26","remote_addr":"[fd00:1122:3344:101::c]:43646","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":105,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:04.597414098Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:57135"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:04.654475369Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"c54669cf-9e94-48a3-9ed1-9e4f75ae867a","remote_addr":"[fd00:1122:3344:101::a]:57135","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":57003,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:04.654651341Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:62923"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:04.7014841Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"e7899a34-95dc-4d2e-b4a0-52f03ab923cc","remote_addr":"[fd00:1122:3344:101::c]:62923","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":46731,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:04.701759903Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"9240c293-7280-4e7a-837e-1fce8fc96868","remote_addr":"[fd00:1122:3344:101::a]:57135","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":112,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:04.701975885Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"369b1116-fd79-4bdf-855f-8f1488917882","remote_addr":"[fd00:1122:3344:101::c]:62923","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":133,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:04.797609425Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:38728"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:04.798516855Z","hostname":"buskin","pid":1555,"uri":"/instances/e15256d7-8018-4d03-a55a-7fe4d0ce46d6/state","method":"GET","req_id":"edc52d0b-b662-4d20-94a9-842e7d892f64","remote_addr":"[fd00:1122:3344:101::b]:38728","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":866,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:04.896814904Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:63740"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:04.896922075Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"806dd17a-648a-40f3-bde8-9ac123caba6e","remote_addr":"[fd00:1122:3344:101::b]:63740","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":71,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:04.908531711Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"5e86e131-29cd-4a40-b399-18b9b96f2000","remote_addr":"[fd00:1122:3344:101::b]:63740","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":132,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:04.987225287Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:50193"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:04.987385938Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"bd118af5-18ea-4eab-ab76-dbade1ccdeec","remote_addr":"[fd00:1122:3344:101::b]:50193","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":137,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:05.091238848Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:35987"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:05.140110619Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"279d8630-2eb1-4481-8d17-e5f7277491e6","remote_addr":"[fd00:1122:3344:101::b]:35987","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":48799,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:05.140465333Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"73d63d69-2d18-4059-a981-0c0dbeddf885","remote_addr":"[fd00:1122:3344:101::b]:35987","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":112,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:05.823202526Z","hostname":"buskin","pid":1555,"uri":"/0dcbb472-7cd1-4e42-9116-fc1cc27da573","method":"GET","req_id":"fedca59d-91a5-49d4-af87-9960a6c30a80","remote_addr":"[fd00:1122:3344:101::d]:59433","local_addr":"[fd00:1122:3344:101::1]:63316","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"0dcbb472-7cd1-4e42-9116-fc1cc27da573","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":610,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:13.155999389Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:49346"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:13.156638796Z","hostname":"buskin","pid":1555,"uri":"/instances/e15256d7-8018-4d03-a55a-7fe4d0ce46d6/state","method":"PUT","req_id":"a7569ec4-fca4-42a1-a552-58983ff33295","remote_addr":"[fd00:1122:3344:101::a]:49346","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":577,"response_code":"200"}
{"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_status: NoMigration, time: 2024-07-11T00:05:13.156734037Z }","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:13.156784807Z","hostname":"buskin","pid":1555,"instance_id":"e15256d7-8018-4d03-a55a-7fe4d0ce46d6","component":"InstanceManager","file":"sled-agent/src/instance.rs:573"}
{"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:13.156807518Z","hostname":"buskin","pid":1555,"instance_id":"e15256d7-8018-4d03-a55a-7fe4d0ce46d6","component":"InstanceManager","file":"sled-agent/src/instance.rs:595","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-07-11T00:05:13.156734037Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(351ac2ad-f19e-403f-9072-cb6eab26571f (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-07-11T00:04:55.176868087Z }","propolis_id":"351ac2ad-f19e-403f-9072-cb6eab26571f"}
{"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:13.156832038Z","hostname":"buskin","pid":1555,"instance_id":"e15256d7-8018-4d03-a55a-7fe4d0ce46d6","component":"InstanceManager","file":"sled-agent/src/instance.rs:492","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(351ac2ad-f19e-403f-9072-cb6eab26571f (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-07-11T00:04:55.176868087Z }, propolis_id: 351ac2ad-f19e-403f-9072-cb6eab26571f (propolis), vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-07-11T00:05:13.156734037Z }, migration_state: None }","instance_id":"e15256d7-8018-4d03-a55a-7fe4d0ce46d6"}
{"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_status: NoMigration, time: 2024-07-11T00:05:13.219571770Z }","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:13.21959394Z","hostname":"buskin","pid":1555,"instance_id":"e15256d7-8018-4d03-a55a-7fe4d0ce46d6","component":"InstanceManager","file":"sled-agent/src/instance.rs:573"}
{"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:13.21961393Z","hostname":"buskin","pid":1555,"instance_id":"e15256d7-8018-4d03-a55a-7fe4d0ce46d6","component":"InstanceManager","file":"sled-agent/src/instance.rs:595","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(6), time_updated: 2024-07-11T00:05:13.219571770Z }","new_instance_state":"InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-07-11T00:05:13.219571770Z }","propolis_id":"351ac2ad-f19e-403f-9072-cb6eab26571f"}
{"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:13.219642391Z","hostname":"buskin","pid":1555,"instance_id":"e15256d7-8018-4d03-a55a-7fe4d0ce46d6","component":"InstanceManager","file":"sled-agent/src/instance.rs:610","instance_id":"e15256d7-8018-4d03-a55a-7fe4d0ce46d6"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:13.219648651Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:13.219834513Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"}
{"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:13.219876963Z","hostname":"buskin","pid":1555,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:437","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/c0d1014c-d647-4991-8139-bcd86ea27898/debug/bundle/zone\", \"/pool/int/cac444c1-8feb-42ab-bee7-aa35319fb578/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/2ff702b9-0076-4c58-8af6-039be08bef86/crypt/debug\", \"/pool/ext/960eb39e-b913-4eb7-b0fb-372e6f629bcb/crypt/debug\", \"/pool/ext/f91400c1-df8c-4cdb-b4f6-cb47c0d105a8/crypt/debug\", \"/pool/ext/d29abc28-1ae7-4192-8340-9766fa8fec22/crypt/debug\", \"/pool/ext/c75afc92-cd90-403e-b6d4-baabf826a6f6/crypt/debug\", \"/pool/ext/e6f9779f-728b-405d-a726-810b70de4ab6/crypt/debug\", \"/pool/ext/2bc49789-2398-4c5f-be70-ff11292cbf60/crypt/debug\", \"/pool/ext/5a2ec449-297e-4e77-989a-dd421ce4d4aa/crypt/debug\", \"/pool/ext/4cab71ff-67df-44db-adca-8ef81d5bc209/crypt/debug\"] }","zone_name":"oxz_propolis-server_351ac2ad-f19e-403f-9072-cb6eab26571f"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:13.257635434Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:55091"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:13.257803856Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"a9123ffa-f9b2-489d-9c25-ea1895f7644d","remote_addr":"[fd00:1122:3344:101::b]:55091","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":106,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:13.269996808Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"66738324-e19a-43f5-aa4c-0dc8ac6b8508","remote_addr":"[fd00:1122:3344:101::b]:55091","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":164,"response_code":"204"}
{"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:13.950450008Z","hostname":"buskin","pid":1555,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1230","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_351ac2ad-f19e-403f-9072-cb6eab26571f\", bundle_id: 12ca32d3-c676-4300-b99d-f94ce5684f21 }, time_created: 2024-07-11T00:05:13.223980108Z, version: 0, cause: TerminatedInstance }"}
{"msg":"Halting and removing zone: oxz_propolis-server_351ac2ad-f19e-403f-9072-cb6eab26571f","v":0,"name":"SledAgent","level":40,"time":"2024-07-11T00:05:13.950557989Z","hostname":"buskin","pid":1555,"instance_id":"e15256d7-8018-4d03-a55a-7fe4d0ce46d6","component":"InstanceManager","file":"sled-agent/src/instance.rs:784"}
{"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:15.732092871Z","hostname":"buskin","pid":1555,"instance_id":"e15256d7-8018-4d03-a55a-7fe4d0ce46d6","component":"InstanceManager","file":"illumos-utils/src/zone.rs:269"}
{"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:15.748691271Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:958","id":"RouterId { vni: Vni(15030357), kind: System }"}
{"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:15.748747132Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:958","id":"RouterId { vni: Vni(15030357), kind: Custom(V4(Ipv4Net { addr: 172.30.0.0, width: 22 })) }"}
{"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:15.758560149Z","hostname":"buskin","pid":1555,"instance_id":"e15256d7-8018-4d03-a55a-7fe4d0ce46d6","component":"InstanceManager","file":"sled-agent/src/instance.rs:492","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-07-11T00:05:13.219571770Z }, propolis_id: 351ac2ad-f19e-403f-9072-cb6eab26571f (propolis), vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-07-11T00:05:15.758492038Z }, migration_state: None }","instance_id":"e15256d7-8018-4d03-a55a-7fe4d0ce46d6"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:15.900497502Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:50926"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:15.900705284Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"463bf9ac-d80a-446e-a98e-13f7d37bd5d8","remote_addr":"[fd00:1122:3344:101::b]:50926","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":174,"response_code":"200"}
{"msg":"Mapping virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:15.900958147Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:832","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 244, 51, 20])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(15030357) }"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:15.900991337Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"PUT","req_id":"06c0742b-f5a8-4d26-93f8-6274964f98b4","remote_addr":"[fd00:1122:3344:101::b]:50926","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":102,"response_code":"204"}
{"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:16.233453713Z","hostname":"buskin","pid":1555,"instance_id":"e15256d7-8018-4d03-a55a-7fe4d0ce46d6","component":"InstanceManager","file":"sled-agent/src/instance.rs:492","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-07-11T00:05:13.219571770Z }, propolis_id: 351ac2ad-f19e-403f-9072-cb6eab26571f (propolis), vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-07-11T00:05:15.758492038Z }, migration_state: None }","instance_id":"e15256d7-8018-4d03-a55a-7fe4d0ce46d6"}
{"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:16.236739828Z","hostname":"buskin","pid":1555,"instance_id":"e15256d7-8018-4d03-a55a-7fe4d0ce46d6","component":"InstanceManager","file":"sled-agent/src/instance.rs:727"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:16.273153274Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:43063"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:16.273263785Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"8f014bf2-d661-4716-9b05-ebd6a23c3ed3","remote_addr":"[fd00:1122:3344:101::b]:43063","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":62,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:16.279641345Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"69b80a08-9c75-494b-90b9-db7ed885c14f","remote_addr":"[fd00:1122:3344:101::b]:43063","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":132,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:16.353080553Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:38277"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:16.353139274Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"6869f8b1-9527-4608-80fd-86b8f6c30dd2","remote_addr":"[fd00:1122:3344:101::b]:38277","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":31,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:16.358918047Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"012a271c-2702-43c7-801b-560be3b0ef8b","remote_addr":"[fd00:1122:3344:101::b]:38277","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":95,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:16.550970375Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:52306"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:16.551199228Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"e7a9c753-8ae7-456d-ae88-60a33529b340","remote_addr":"[fd00:1122:3344:101::a]:52306","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":202,"response_code":"200"}
{"msg":"Clearing mapping of virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:16.551492201Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:872","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 244, 51, 20])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(15030357) }"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:16.551526631Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"DELETE","req_id":"0afcced5-561c-4476-a07e-dec9d7c57796","remote_addr":"[fd00:1122:3344:101::a]:52306","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":123,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:17.86287527Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:63614"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:17.863453456Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"bf886955-2c0a-4c02-8eb5-1b80664289a8","remote_addr":"[fd00:1122:3344:101::a]:63614","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":560,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-11T00:05:17.86937726Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"4ac90a5a-8089-4d77-9281-f0674e7afb43","remote_addr":"[fd00:1122:3344:101::a]:63614","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":126,"response_code":"204"}