[ Jul 6 00:02:49 Enabled. ] [ Jul 6 00:02:49 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Jul 6 00:02:49 Method "start" exited with status 0. ] [ Jul 6 00:02:49 Rereading configuration. ] [ Jul 6 00:02:49 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:49.320083727Z","hostname":"sock","pid":1555,"file":"illumos-utils/src/running_zone.rs:163"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:49.32030532Z","hostname":"sock","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:49.372349131Z","hostname":"sock","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:49.372393452Z","hostname":"sock","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:49.488492456Z","hostname":"sock","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:49.497088603Z","hostname":"sock","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:49.505093251Z","hostname":"sock","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.107921876Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fdb0:18c0:4d0d:a02a::, width: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.250876029Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:119"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.25094409Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:131"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.25096097Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:141"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.251016331Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:162","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.251055371Z","hostname":"sock","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:703"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.251073172Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:506"} {"msg":"Generated i86pc baseboard Pc { identifier: \"sock\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.25408936Z","hostname":"sock","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:610"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:02:50.25409656Z","hostname":"sock","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:752"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.25411044Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:176"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.25411532Z","hostname":"sock","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:811"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.254148831Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.254250262Z","hostname":"sock","pid":1555,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.254263183Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.254290343Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:495","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.254300813Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.2660056Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.266040311Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:372","id":"f8eb5131-e7a0-4521-9f31-d994085bd770 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.365023951Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: f8eb5131-e7a0-4521-9f31-d994085bd770 (zpool), kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.572423473Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.572455384Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.572462013Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:564","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.572478164Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:585","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.572483584Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.572492624Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:495","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.572512154Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.580899888Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.580926259Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:372","id":"fea674a6-6f32-4040-ae6a-1581a3ccf192 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.673001539Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: fea674a6-6f32-4040-ae6a-1581a3ccf192 (zpool), kind: Internal }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875031065Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_0.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875072845Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: f8eb5131-e7a0-4521-9f31-d994085bd770 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: fea674a6-6f32-4040-ae6a-1581a3ccf192 (zpool), kind: Internal } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875106166Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875114516Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875124476Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875158297Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:495","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875191238Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_1.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875201098Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: f8eb5131-e7a0-4521-9f31-d994085bd770 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: fea674a6-6f32-4040-ae6a-1581a3ccf192 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875213268Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875220668Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875265369Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875272389Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:495","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875312689Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_2.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.87533825Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.87534504Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:495","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.87534949Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: f8eb5131-e7a0-4521-9f31-d994085bd770 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: fea674a6-6f32-4040-ae6a-1581a3ccf192 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.87536887Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.87537342Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875378221Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_3.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.87538297Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875388941Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:495","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875393001Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: f8eb5131-e7a0-4521-9f31-d994085bd770 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: fea674a6-6f32-4040-ae6a-1581a3ccf192 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875415021Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875419861Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875423071Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_4.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875427291Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: f8eb5131-e7a0-4521-9f31-d994085bd770 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: fea674a6-6f32-4040-ae6a-1581a3ccf192 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875444322Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875448971Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875452902Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875457962Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:495","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875462032Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_5.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875466912Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: f8eb5131-e7a0-4521-9f31-d994085bd770 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: fea674a6-6f32-4040-ae6a-1581a3ccf192 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875491612Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875496062Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875500983Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875504752Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:495","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875511223Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_6.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875524483Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: f8eb5131-e7a0-4521-9f31-d994085bd770 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: fea674a6-6f32-4040-ae6a-1581a3ccf192 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875545893Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875558423Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875562643Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875566253Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:495","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875570224Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_7.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875578834Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875583744Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:495","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875587914Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: f8eb5131-e7a0-4521-9f31-d994085bd770 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: fea674a6-6f32-4040-ae6a-1581a3ccf192 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875609524Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875614184Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875630174Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_8.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875634534Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875638375Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:495","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875642885Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: f8eb5131-e7a0-4521-9f31-d994085bd770 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: fea674a6-6f32-4040-ae6a-1581a3ccf192 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875672295Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875677035Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875680435Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:91"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875690775Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:93"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875694395Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: f8eb5131-e7a0-4521-9f31-d994085bd770 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: fea674a6-6f32-4040-ae6a-1581a3ccf192 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875724636Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875729606Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875733266Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875736756Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:200"} {"msg":"No ledger in /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875798797Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875813537Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875824517Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875836228Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875844418Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875849008Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875878318Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:205"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.875882629Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:220"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.886773422Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:712"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.886824483Z","hostname":"sock","pid":1555,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:289","time_to_next_cleanup":"599.999923349s","next_cleanup":"Instant { tv_sec: 2149, tv_nsec: 867667599 }"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.886881494Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3568"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.886904954Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"No ledger in /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.886970106Z","hostname":"sock","pid":1555,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.888685373Z","hostname":"sock","pid":1555,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.889814821Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:196"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.889896652Z","hostname":"sock","pid":1555,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:44","local_addr":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.2.164,\n last: 10.151.2.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.2.164,\n 10.151.2.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.2.174,\n infra_ip_last: 10.151.2.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.2.100,\n vlan_id: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.2.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.890167277Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1011"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.890172517Z","hostname":"sock","pid":1555,"component":"DnsResolver","component":"RSS","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.890217617Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"No ledger in /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.890272948Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.890309739Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.890317999Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1071"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.890346589Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"No ledger in /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.89036658Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.89037442Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.89038024Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1112"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.89038342Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:156"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:a02a::1","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.89038664Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:129"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.89041595Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Writing ledger to /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.890443751Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.891186453Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Sled plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.891346756Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:176"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.891362306Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1145"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.891414817Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.891421737Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.891580369Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.891694951Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.891736122Z","hostname":"sock","pid":1555,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:217","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 4d88c6ce-c5a2-418b-b915-09fea24f811c, rack_id: 444442ff-30f3-4f30-9be3-25f64879f808, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.891866344Z","hostname":"sock","pid":1555,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:38943","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:66"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 4d88c6ce-c5a2-418b-b915-09fea24f811c, rack_id: 444442ff-30f3-4f30-9be3-25f64879f808, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.891990556Z","hostname":"sock","pid":1555,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:360"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.892007106Z","hostname":"sock","pid":1555,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:375"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.892015546Z","hostname":"sock","pid":1555,"file":"sled-agent/src/server.rs:44"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.892032686Z","hostname":"sock","pid":1555,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.892061047Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.892068947Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:574"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.892095147Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:101::, width: 64 }"} {"msg":"No ledger in /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.892101957Z","hostname":"sock","pid":1555,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.892114018Z","hostname":"sock","pid":1555,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.892125308Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:563"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.892129778Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:591"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.894733619Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3800"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.902313431Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:647","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.912086196Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:647","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.923976257Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:647","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.933430937Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:647","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.942100156Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:647","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.950412959Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:647","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.958725381Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:647","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.966947293Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:647","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.975178223Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:647","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"No disks to be automatically managed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.975218664Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:602"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.975223574Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.975228084Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:395"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.975249034Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:340"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.975258204Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:481","result":"DisksManagementResult { status: [] }"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.986549705Z","hostname":"sock","pid":1555,"sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:355"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.986598196Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.986622786Z","hostname":"sock","pid":1555,"sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:367"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.986679847Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 96.383329ms","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.992345657Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0c880841-1f99-4a6a-a0ea-91d17facfb14\", \"content-length\": \"124\", \"date\": \"Sat, 06 Jul 2024 00:02:50 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0c880841-1f99-4a6a-a0ea-91d17facfb14\" }"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.997044482Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"attempting to create encrypted zvol: name=\"oxi_f8eb5131-e7a0-4521-9f31-d994085bd770/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:50.999701945Z","hostname":"sock","pid":1555,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:51.014625233Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"successfully created encrypted zvol: name=\"oxi_f8eb5131-e7a0-4521-9f31-d994085bd770/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:51.023975262Z","hostname":"sock","pid":1555,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_f8eb5131-e7a0-4521-9f31-d994085bd770/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:51.024028333Z","hostname":"sock","pid":1555,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:51.028916781Z","hostname":"sock","pid":1555,"sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:382"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:51.028925261Z","hostname":"sock","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:126"} {"msg":"Ensuring dataset oxi_f8eb5131-e7a0-4521-9f31-d994085bd770/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:51.028931161Z","hostname":"sock","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:136"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:51.08959333Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 110.502548ms","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:51.090167359Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"53c09128-ff21-4890-98c4-15ab2d88fda5\", \"content-length\": \"124\", \"date\": \"Sat, 06 Jul 2024 00:02:50 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"53c09128-ff21-4890-98c4-15ab2d88fda5\" }"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:51.100555895Z","hostname":"sock","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:170"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:51.201727031Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 181.031562ms","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:51.202264059Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3c97e82e-874c-406e-b43a-5c42dab48bd6\", \"content-length\": \"124\", \"date\": \"Sat, 06 Jul 2024 00:02:51 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3c97e82e-874c-406e-b43a-5c42dab48bd6\" }"} {"msg":"Mounting oxi_f8eb5131-e7a0-4521-9f31-d994085bd770/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:51.319014453Z","hostname":"sock","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:178"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:51.33320672Z","hostname":"sock","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:51.333284101Z","hostname":"sock","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:51.333297952Z","hostname":"sock","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:51.333314662Z","hostname":"sock","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:192"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:51.383580335Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 201.280479ms","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:51.384251575Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"356ffe29-a6f7-499f-8514-e8924c1a9d40\", \"content-length\": \"124\", \"date\": \"Sat, 06 Jul 2024 00:02:51 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"356ffe29-a6f7-499f-8514-e8924c1a9d40\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:51.587507281Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 200.008258ms","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:51.588343564Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c16e5795-2a29-43fd-a581-155c3f7291c0\", \"content-length\": \"124\", \"date\": \"Sat, 06 Jul 2024 00:02:51 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c16e5795-2a29-43fd-a581-155c3f7291c0\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:51.788529281Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 126.96308ms","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:51.789191231Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b11ac0bb-530c-4816-95e4-572f089c3c07\", \"content-length\": \"124\", \"date\": \"Sat, 06 Jul 2024 00:02:51 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b11ac0bb-530c-4816-95e4-572f089c3c07\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:51.917238766Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 157.989739ms","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:51.917696113Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"90c85fad-4fec-4d7c-8465-97d57b251882\", \"content-length\": \"124\", \"date\": \"Sat, 06 Jul 2024 00:02:51 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"90c85fad-4fec-4d7c-8465-97d57b251882\" }"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:52.037218692Z","hostname":"sock","pid":1555,"sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:90"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:52.038050285Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:56314","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:196","component":"producer-server","component":"MetricsManager","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:196"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:52.038072805Z","hostname":"sock","pid":1555,"component":"internal-dns-resolver","file":"internal-dns/src/resolver.rs:60","component":"producer-server","component":"MetricsManager","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:52.038137856Z","hostname":"sock","pid":1555,"file":"oximeter/producer/src/lib.rs:282","component":"producer-server","component":"MetricsManager","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:282","interval":"30s","address":"[fd00:1122:3344:101::1]:56314","producer_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:02:52.044402856Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:390","component":"producer-server","component":"MetricsManager","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:390","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"279.281921ms"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:52.076790904Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:52.085046425Z","hostname":"sock","pid":1555,"component":"VmmReservoirManager","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:52.085072756Z","hostname":"sock","pid":1555,"component":"VmmReservoirManager","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:52.085197468Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:895","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:52.085207668Z","hostname":"sock","pid":1555,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:52.086196214Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:196"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:52.086223954Z","hostname":"sock","pid":1555,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:430"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:52.086251605Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Writing ledger to /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:52.086266875Z","hostname":"sock","pid":1555,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:52.086502549Z","hostname":"sock","pid":1555,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:52.086645291Z","hostname":"sock","pid":1555,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:52.086672581Z","hostname":"sock","pid":1555,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV2 { rack_subnet: Ipv6Net { addr: fd00:1122:3344:100::, width: 56 }, infra_ip_first: 10.151.2.174, infra_ip_last: 10.151.2.174, ports: [PortConfigV2 { routes: [RouteConfig { destination: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), nexthop: 10.151.2.100, vlan_id: None }], addresses: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false }], bgp: [], bfd: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:52.086685002Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3568"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:52.086690532Z","hostname":"sock","pid":1555,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:38943","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:73"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:52.086744112Z","hostname":"sock","pid":1555,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:108","peer":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:52.086750352Z","hostname":"sock","pid":1555,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:52.298024906Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"No ledger in /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:52.298111007Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:52.298130818Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:52.298145718Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:52.298161848Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/4d88c6ce-c5a2-418b-b915-09fea24f811c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:02:52.303067807Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:02:52.325996003Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:390","component":"producer-server","component":"MetricsManager","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:390","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"703.164996ms"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:52.329491889Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:33769"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:52.329658651Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:52.329756053Z","hostname":"sock","pid":1555,"uri":"/inventory","method":"GET","req_id":"2ad2c491-5228-4841-9a3b-dddf7be23e65","remote_addr":"[fd00:1122:3344:101::1]:33769","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":165,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:52.357977254Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:62576"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:52.358068895Z","hostname":"sock","pid":1555,"uri":"/sled-role","method":"GET","req_id":"9b550a1e-3077-422f-879a-699ddda380fc","remote_addr":"[fd00:1122:3344:101::1]:62576","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":32,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:52.359526688Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Writing ledger to /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:52.359556829Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:52.359899804Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:52.360178149Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:832"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { disks: [OmicronPhysicalDiskConfig { id: 823ccb53-564e-4856-a2c4-ebec194b623b, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, pool_id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool) }, OmicronPhysicalDiskConfig { id: bc7c5da9-6a4e-470b-9728-bd85df9e1cd8, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, pool_id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool) }, OmicronPhysicalDiskConfig { id: c267bc4e-aeb0-40a5-ad1e-4359f88ae781, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, pool_id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool) }, OmicronPhysicalDiskConfig { id: 5c3a7a47-457a-41cd-b5da-13dc7bde6f51, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, pool_id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool) }, OmicronPhysicalDiskConfig { id: 37c51920-d2bb-432d-998c-e1fa5cc90b6c, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, pool_id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool) }, OmicronPhysicalDiskConfig { id: 8977113b-f3f6-414e-9dd1-542f38f0a21e, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, pool_id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool) }, OmicronPhysicalDiskConfig { id: 391d1448-2715-499c-908e-400c87ee9d62, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, pool_id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool) }, OmicronPhysicalDiskConfig { id: 0ea77ba5-6362-413c-9740-c09158313166, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, pool_id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool) }, OmicronPhysicalDiskConfig { id: ae4373cb-5d70-4f28-a221-55d181604379, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, pool_id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool) }], generation: Generation(1) }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:52.38908591Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:348"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:52.389219722Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:37223"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 823ccb53-564e-4856-a2c4-ebec194b623b, pool_id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: bc7c5da9-6a4e-470b-9728-bd85df9e1cd8, pool_id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: c267bc4e-aeb0-40a5-ad1e-4359f88ae781, pool_id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 5c3a7a47-457a-41cd-b5da-13dc7bde6f51, pool_id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 37c51920-d2bb-432d-998c-e1fa5cc90b6c, pool_id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: 8977113b-f3f6-414e-9dd1-542f38f0a21e, pool_id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 391d1448-2715-499c-908e-400c87ee9d62, pool_id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 0ea77ba5-6362-413c-9740-c09158313166, pool_id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: ae4373cb-5d70-4f28-a221-55d181604379, pool_id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:52.389495307Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"No ledger in /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:52.389501337Z","hostname":"sock","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:52.389517137Z","hostname":"sock","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:52.389522177Z","hostname":"sock","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:725"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:52.389573858Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:340"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:52.389583188Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:380","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:52.389588838Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:457"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:52.389593458Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:52.515812014Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:52.515862164Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"9e1edfda-939b-489d-a7cb-446b4599864c (zpool)"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:02:53.138010328Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:390","component":"producer-server","component":"MetricsManager","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:390","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"1.389439267s"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/4d88c6ce-c5a2-418b-b915-09fea24f811c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-06T00:02:53.138362894Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:171"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:53.504075903Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:53.683106722Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:53.683159713Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:53.683168213Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:53.683177503Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:53.683204884Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:53.683323196Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:53.683332196Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:54.228542131Z","hostname":"sock","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:54.431301468Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/4d88c6ce-c5a2-418b-b915-09fea24f811c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:02:54.845804617Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:02:54.845869828Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:390","component":"producer-server","component":"MetricsManager","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:390","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"2.645815301s"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/4d88c6ce-c5a2-418b-b915-09fea24f811c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-06T00:02:54.846046991Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:171"} {"msg":"Automatically destroying dataset: oxp_9e1edfda-939b-489d-a7cb-446b4599864c/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:54.846161423Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 32.352337ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:02:55.022181233Z","hostname":"sock","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 131.144631ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:02:55.546559276Z","hostname":"sock","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 101.265405ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:02:55.949168725Z","hostname":"sock","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/4d88c6ce-c5a2-418b-b915-09fea24f811c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-06T00:02:56.153770792Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:171"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 354.036185ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:02:56.332853871Z","hostname":"sock","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/4d88c6ce-c5a2-418b-b915-09fea24f811c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:02:56.334452987Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 410.386022ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:02:57.053316155Z","hostname":"sock","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/4d88c6ce-c5a2-418b-b915-09fea24f811c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-06T00:02:57.251107473Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:171"} {"msg":"Looking for unencrypted datasets in oxp_9e1edfda-939b-489d-a7cb-446b4599864c","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:57.622072597Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.064852183s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:02:57.845478534Z","hostname":"sock","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:02:57.848021485Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:390","component":"producer-server","component":"MetricsManager","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:390","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"2.554966125s"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:57.855416253Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:478","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:57.855427703Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:395","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:57.855449953Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:380","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:57.855454613Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:457"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:57.855461554Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:58.0957281Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:58.095770661Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:59.354894015Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:59.599242157Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:59.599290038Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:59.599296648Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:59.599304738Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:59.599310398Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:59.59943202Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:02:59.59944145Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/4d88c6ce-c5a2-418b-b915-09fea24f811c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-06T00:03:00.095976088Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:171"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/4d88c6ce-c5a2-418b-b915-09fea24f811c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:00.096041439Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:00.81507476Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:390","component":"producer-server","component":"MetricsManager","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:390","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"8.880434936s"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:01.068404216Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/4d88c6ce-c5a2-418b-b915-09fea24f811c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-06T00:03:01.583002673Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:171"} {"msg":"Automatically destroying dataset: oxp_c029673f-d8f0-4d66-9f90-dcc8d5377a3a/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:01.583331898Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/4d88c6ce-c5a2-418b-b915-09fea24f811c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:02.057934736Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:02.273996846Z","hostname":"sock","pid":1555,"component":"VmmReservoirManager","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"} {"msg":"Looking for unencrypted datasets in oxp_c029673f-d8f0-4d66-9f90-dcc8d5377a3a","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:02.387201993Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:02.395094569Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:478","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:02.39515043Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:395","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:02.39515704Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:380","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:02.395162621Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:457"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:02.395167481Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:02.404336737Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:02.404363897Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:02.499757911Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","zpool":"ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:02.508059503Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:02.508088364Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:02.508093494Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:02.508113094Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:02.508120944Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:02.508239156Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:02.508252976Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:02.546365625Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_187c3b21-21e5-4726-9974-0290b3e4dbd4/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:02.554476684Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/4d88c6ce-c5a2-418b-b915-09fea24f811c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-06T00:03:02.585129504Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:171"} {"msg":"Looking for unencrypted datasets in oxp_187c3b21-21e5-4726-9974-0290b3e4dbd4","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:02.679124555Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:02.687160993Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:478","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:02.687187393Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:395","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:02.687192624Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:380","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:02.687198034Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:457"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:02.687202044Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:02.696538493Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:02.696569373Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:02.791011391Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","zpool":"ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:02.799381485Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:02.799412985Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:02.799419376Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:02.799429006Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:02.799435976Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:02.799562198Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:02.799568148Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:02.833835355Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_69edb71d-4c73-4b0c-a984-e973915bfc3d/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:02.842391732Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_69edb71d-4c73-4b0c-a984-e973915bfc3d","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:02.964514132Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:02.972330776Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:478","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:02.972357186Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:395","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:02.972362966Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:380","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:02.972374797Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:457"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:02.972378877Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:02.981829588Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:02.981857468Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"05665420-5e42-4534-83d1-b4b73f92c491 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.073361379Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","zpool":"ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.081759163Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.081789873Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.081796134Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.081827734Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.081836114Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.082014877Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.082024327Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.117632926Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_05665420-5e42-4534-83d1-b4b73f92c491/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.125852357Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_05665420-5e42-4534-83d1-b4b73f92c491","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.248821571Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.257096463Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:478","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.257127683Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:395","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.257148244Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:380","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.257152914Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:457"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.257160114Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.26692343Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.26695078Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.36278778Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","zpool":"ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.370754398Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.370783898Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.370789228Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.370814209Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.370825609Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.370987022Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.370996462Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.406310395Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_1fb8d37a-fccb-4d16-a337-6873d3def608/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.414596178Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_1fb8d37a-fccb-4d16-a337-6873d3def608","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.536763678Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.544689255Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:478","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.544715356Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:395","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.544721296Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:380","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.544725906Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:457"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.544730966Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.553979773Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.554018974Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.653554283Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","zpool":"ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.660541075Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.660574915Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.660580246Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.660603796Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.660614556Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.660791339Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.660797219Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/4d88c6ce-c5a2-418b-b915-09fea24f811c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-06T00:03:03.664426317Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:171"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.695723817Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_03b41f7a-86c6-414e-8276-ffa92bf542a8/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.703008563Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_03b41f7a-86c6-414e-8276-ffa92bf542a8","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.82619309Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.833951464Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:478","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.834010175Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:395","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.834021765Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:380","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.834030015Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:457"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.834036515Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.843988424Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.844033065Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool)"} {"msg":"Ensuring tfportrear0_0/ll exists in zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.855524888Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2244"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.9451927Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","zpool":"ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.953600894Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.953643595Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.953651255Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.953670186Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.953699616Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.953866339Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.953873499Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.989580349Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_d0e51e3d-0355-4324-9d0e-c910d0a56ce1/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:03.997557536Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/4d88c6ce-c5a2-418b-b915-09fea24f811c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:04.061216093Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Looking for unencrypted datasets in oxp_d0e51e3d-0355-4324-9d0e-c910d0a56ce1","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:04.122241777Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:04.130612851Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:478","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:04.130643312Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:395","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:04.130649121Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:380","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:04.130654382Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:457"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:04.130660512Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:04.140226434Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:04.140256405Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:04.237160272Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","zpool":"ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:04.24515478Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:04.24517994Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:04.245185841Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:04.245244521Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:04.245251262Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:04.245383844Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:04.245389684Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:04.280725438Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_af3bec9c-e31f-4896-95d7-21d956ea9d1e/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:04.288879538Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_af3bec9c-e31f-4896-95d7-21d956ea9d1e","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:04.416387524Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:04.424581215Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:478","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:04.424634596Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:395","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Writing ledger to /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:04.424672296Z","hostname":"sock","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:04.424832079Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: f8eb5131-e7a0-4521-9f31-d994085bd770 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: fea674a6-6f32-4040-ae6a-1581a3ccf192 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:04.42487502Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:04.42488449Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Writing ledger to /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:04.425004942Z","hostname":"sock","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:04.425400248Z","hostname":"sock","pid":1555,"uri":"/omicron-physical-disks","method":"PUT","req_id":"6e391d4e-2cb9-4ae5-b352-2ad2f0c73201","remote_addr":"[fd00:1122:3344:101::1]:37223","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":12036120,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: cdec4b62-7244-4b80-8777-6d905a5202cd, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:04.45557993Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:04.455701952Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:47050"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:04.455947816Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:04.455953226Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:04.506715586Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:04.506749957Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:04.557387606Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:04.557417366Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:04.607583727Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"No ledger in /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:04.607615918Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:04.607629308Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:04.607680869Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:04.653049443Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:04.653090284Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:04.653098844Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/4d88c6ce-c5a2-418b-b915-09fea24f811c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-06T00:03:04.666044171Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:171"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_bd5693ce-2405-4f5e-bcea-cf00f7435aea","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:04.705137005Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_1f295afe-8b02-48b5-8b08-e8e6206ee96c","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:04.708905465Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_cdec4b62-7244-4b80-8777-6d905a5202cd","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:04.712895509Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_internal_dns_bd5693ce-2405-4f5e-bcea-cf00f7435aea","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:04.724924981Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_1f295afe-8b02-48b5-8b08-e8e6206ee96c","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:04.729763508Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_cdec4b62-7244-4b80-8777-6d905a5202cd","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:04.733767912Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/4d88c6ce-c5a2-418b-b915-09fea24f811c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-06T00:03:05.668511497Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:171"} {"msg":"Profile for oxz_internal_dns_cdec4b62-7244-4b80-8777-6d905a5202cd:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:06.259202588Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:06.2592659Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:1::, width: 64 }"} {"msg":"Profile for oxz_internal_dns_bd5693ce-2405-4f5e-bcea-cf00f7435aea:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:06.740142988Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:07.218382604Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:2::, width: 64 }"} {"msg":"Profile for oxz_internal_dns_1f295afe-8b02-48b5-8b08-e8e6206ee96c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:07.218465296Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:07.218515806Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_cdec4b62-7244-4b80-8777-6d905a5202cd","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:07.222170115Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:3::, width: 64 }"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:07.222195795Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_bd5693ce-2405-4f5e-bcea-cf00f7435aea","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:07.22685551Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_1f295afe-8b02-48b5-8b08-e8e6206ee96c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_internal_dns_1f295afe-8b02-48b5-8b08-e8e6206ee96c\") failed: Property not found. retry in 74.030222ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:07.345662747Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_1f295afe-8b02-48b5-8b08-e8e6206ee96c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_cdec4b62-7244-4b80-8777-6d905a5202cd\") failed: Property not found. retry in 50.766431ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:07.351783774Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_cdec4b62-7244-4b80-8777-6d905a5202cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_bd5693ce-2405-4f5e-bcea-cf00f7435aea\") failed: Property not found. retry in 35.923207ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:07.35773804Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_bd5693ce-2405-4f5e-bcea-cf00f7435aea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_bd5693ce-2405-4f5e-bcea-cf00f7435aea\") failed: Property not found. retry in 126.54684ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:07.400462522Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_bd5693ce-2405-4f5e-bcea-cf00f7435aea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_cdec4b62-7244-4b80-8777-6d905a5202cd\") failed: Property not found. retry in 104.206095ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:07.428087183Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_cdec4b62-7244-4b80-8777-6d905a5202cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_1f295afe-8b02-48b5-8b08-e8e6206ee96c\") failed: Property not found. retry in 98.487892ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:07.532447249Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_1f295afe-8b02-48b5-8b08-e8e6206ee96c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_bd5693ce-2405-4f5e-bcea-cf00f7435aea\") failed: Property not found. retry in 243.969646ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:07.691452288Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_bd5693ce-2405-4f5e-bcea-cf00f7435aea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_1f295afe-8b02-48b5-8b08-e8e6206ee96c\") failed: Property not found. retry in 196.091015ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:07.727986742Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_1f295afe-8b02-48b5-8b08-e8e6206ee96c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_cdec4b62-7244-4b80-8777-6d905a5202cd\") failed: Property not found. retry in 239.965996ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:07.771765361Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_cdec4b62-7244-4b80-8777-6d905a5202cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_1f295afe-8b02-48b5-8b08-e8e6206ee96c\") failed: Property not found. retry in 221.435533ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:07.977670098Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_1f295afe-8b02-48b5-8b08-e8e6206ee96c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_bd5693ce-2405-4f5e-bcea-cf00f7435aea\") failed: Property not found. retry in 515.948425ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:08.094516524Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_bd5693ce-2405-4f5e-bcea-cf00f7435aea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_cdec4b62-7244-4b80-8777-6d905a5202cd\") failed: Property not found. retry in 318.261232ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:08.128630888Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_cdec4b62-7244-4b80-8777-6d905a5202cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_1f295afe-8b02-48b5-8b08-e8e6206ee96c\") failed: Property not found. retry in 747.136876ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:08.209131124Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_1f295afe-8b02-48b5-8b08-e8e6206ee96c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_cdec4b62-7244-4b80-8777-6d905a5202cd\") failed: Property not found. retry in 461.122145ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:08.456950601Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_cdec4b62-7244-4b80-8777-6d905a5202cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_bd5693ce-2405-4f5e-bcea-cf00f7435aea\") failed: Property not found. retry in 1.065849244s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:08.620040895Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_bd5693ce-2405-4f5e-bcea-cf00f7435aea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_cdec4b62-7244-4b80-8777-6d905a5202cd\") failed: Property not found. retry in 1.011809713s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:08.929925553Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_cdec4b62-7244-4b80-8777-6d905a5202cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_1f295afe-8b02-48b5-8b08-e8e6206ee96c\") failed: Property not found. retry in 674.926371ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:08.965694755Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_1f295afe-8b02-48b5-8b08-e8e6206ee96c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_1f295afe-8b02-48b5-8b08-e8e6206ee96c\") failed: Property not found. retry in 586.864153ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:09.652806415Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_1f295afe-8b02-48b5-8b08-e8e6206ee96c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_bd5693ce-2405-4f5e-bcea-cf00f7435aea\") failed: Property not found. retry in 557.779121ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:09.696190778Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_bd5693ce-2405-4f5e-bcea-cf00f7435aea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_cdec4b62-7244-4b80-8777-6d905a5202cd\") failed: Property not found. retry in 910.721372ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:09.951940792Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_cdec4b62-7244-4b80-8777-6d905a5202cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring bootstrap address fdb0:18c0:4d0d:a02a::2 exists in switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:09.9693508Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2261"} {"msg":"Adding bootstrap address","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:09.96936806Z","hostname":"sock","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:626"} {"msg":"wait for service Some(\"oxz_internal_dns_1f295afe-8b02-48b5-8b08-e8e6206ee96c\") failed: Property not found. retry in 634.505069ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:10.248821392Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_1f295afe-8b02-48b5-8b08-e8e6206ee96c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_bd5693ce-2405-4f5e-bcea-cf00f7435aea\") failed: Property not found. retry in 1.491466938s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:10.261995073Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_bd5693ce-2405-4f5e-bcea-cf00f7435aea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Forwarding bootstrap traffic via oxBootstrap0 to fe80::8:20ff:feb4:d085","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:10.524400643Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2268"} {"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:10.530001392Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2329"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:10.650017689Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2514"} {"msg":"wait for service Some(\"oxz_internal_dns_cdec4b62-7244-4b80-8777-6d905a5202cd\") failed: Property not found. retry in 500.645452ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:10.871694349Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_cdec4b62-7244-4b80-8777-6d905a5202cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_1f295afe-8b02-48b5-8b08-e8e6206ee96c\") failed: Property not found. retry in 954.107476ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:10.890366727Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_1f295afe-8b02-48b5-8b08-e8e6206ee96c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:10.894006685Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2678"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:11.081540299Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2393"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:11.375606534Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2428"} {"msg":"wait for service Some(\"oxz_internal_dns_cdec4b62-7244-4b80-8777-6d905a5202cd\") failed: Property not found. retry in 906.127843ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:11.381270265Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_cdec4b62-7244-4b80-8777-6d905a5202cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:11.574453569Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2728"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:11.749340312Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2755"} {"msg":"wait for service Some(\"oxz_internal_dns_bd5693ce-2405-4f5e-bcea-cf00f7435aea\") failed: Property not found. retry in 1.419783049s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:11.763286085Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_bd5693ce-2405-4f5e-bcea-cf00f7435aea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_1f295afe-8b02-48b5-8b08-e8e6206ee96c\") failed: Property not found. retry in 530.629219ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:11.855246993Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_1f295afe-8b02-48b5-8b08-e8e6206ee96c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:11.967898672Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2622"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:12.152790493Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2422"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:12.162947826Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3824","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:12.162986376Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3840"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:12.162997066Z","hostname":"sock","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"wait for service Some(\"oxz_internal_dns_cdec4b62-7244-4b80-8777-6d905a5202cd\") failed: Property not found. retry in 516.61681ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:12.297350101Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_cdec4b62-7244-4b80-8777-6d905a5202cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_1f295afe-8b02-48b5-8b08-e8e6206ee96c\") failed: Property not found. retry in 856.472289ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:12.394889129Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_1f295afe-8b02-48b5-8b08-e8e6206ee96c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:12.717341528Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3848"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:12.724079695Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3904"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:12.887471744Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3966"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.012203426Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3947","rack_subnet":"fd00:1122:3344::"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.040031251Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3994"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.163301829Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4021"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.271495216Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_internal_dns_bd5693ce-2405-4f5e-bcea-cf00f7435aea"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.271555617Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_internal_dns_1f295afe-8b02-48b5-8b08-e8e6206ee96c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.271582388Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_internal_dns_cdec4b62-7244-4b80-8777-6d905a5202cd"} {"msg":"Writing ledger to /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.271593538Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.271864642Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.272116217Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"d566ee79-a4d2-48d8-bf7f-59734e901c59","remote_addr":"[fd00:1122:3344:101::1]:47050","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":8816328,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.273750012Z","hostname":"sock","pid":1555,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:549"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.298987015Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:369"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.341256861Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:417"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.372181834Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:656"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.372267886Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:694"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.373384154Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:697","uptime":"3"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.373399824Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:439","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.2.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n}"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.448148067Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.448241709Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:564","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.448258549Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:585","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.448268169Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:564","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.448275179Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:585","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.44828047Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:564","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.448287719Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:585","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.44829426Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:564","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.44830096Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:585","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.44830785Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:564","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.44831648Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:585","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.44832363Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:564","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.4483308Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:585","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.44833784Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:564","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.44835262Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:585","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.448374921Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:564","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.448382241Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:585","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.448389471Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:564","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.448396461Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:585","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.448403431Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:564","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.448410781Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:585","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.448417482Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:564","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.448424062Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:585","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"failed to write DNS configuration (will retry in 97.068456ms)","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:13.550315109Z","hostname":"sock","pid":1555,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:569","error_message":"Communication Error: error sending request for url (http://[fd00:1122:3344:1::1]:5353/config): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.667253636Z","hostname":"sock","pid":1555,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:549"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.712818363Z","hostname":"sock","pid":1555,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:549"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.76020184Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:585"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.76022268Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.791558581Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.793098275Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.793104256Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: cdec4b62-7244-4b80-8777-6d905a5202cd, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 92bc3f3a-c64a-4323-b745-aba7d1b54a8d, kind: Service { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a }, name: Name(\"ntp-af96c243-6c6a-4cb4-8eab-8f8a25fd030a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 160])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.821032351Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.821164393Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:43376"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.821394897Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.821426088Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.842899961Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.842942151Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.863106853Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.863146154Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.882441342Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.882498143Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.882691186Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.882834378Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.894150659Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.92554549Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.926204201Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.926221031Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.970994845Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:478","port":"Port { inner: PortInner(PortData { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 254, 160]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 }, vnic: \"vopte0\" }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.971049446Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1200","zone_type":"ntp"} {"msg":"no nat entry found for: 10.151.2.169","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.971678046Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.169","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.971686666Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.2.169","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.97194607Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.972990917Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Configuring new Omicron zone: oxz_ntp_af96c243-6c6a-4cb4-8eab-8f8a25fd030a","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:13.995367284Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_ntp_af96c243-6c6a-4cb4-8eab-8f8a25fd030a","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:14.0151002Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_ntp_af96c243-6c6a-4cb4-8eab-8f8a25fd030a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:14.634226226Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:14.634304787Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_af96c243-6c6a-4cb4-8eab-8f8a25fd030a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_ntp_af96c243-6c6a-4cb4-8eab-8f8a25fd030a\") failed: Property not found. retry in 25.219296ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:14.77106324Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_af96c243-6c6a-4cb4-8eab-8f8a25fd030a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_af96c243-6c6a-4cb4-8eab-8f8a25fd030a\") failed: Property not found. retry in 68.225958ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:14.804097888Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_af96c243-6c6a-4cb4-8eab-8f8a25fd030a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_af96c243-6c6a-4cb4-8eab-8f8a25fd030a\") failed: Property not found. retry in 298.229442ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:14.885393806Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_af96c243-6c6a-4cb4-8eab-8f8a25fd030a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_af96c243-6c6a-4cb4-8eab-8f8a25fd030a\") failed: Property not found. retry in 379.244797ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:15.194196416Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_af96c243-6c6a-4cb4-8eab-8f8a25fd030a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_af96c243-6c6a-4cb4-8eab-8f8a25fd030a\") failed: Property not found. retry in 1.196940895s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:15.582750151Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_af96c243-6c6a-4cb4-8eab-8f8a25fd030a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:16.654660886Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:55683"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:16.654790818Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"b498d50e-7aa6-4ae3-a02c-57794afb2f4f","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:55683","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":60,"response_code":"200"} {"msg":"wait for service Some(\"oxz_ntp_af96c243-6c6a-4cb4-8eab-8f8a25fd030a\") failed: Property not found. retry in 1.202213205s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:16.789902955Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_af96c243-6c6a-4cb4-8eab-8f8a25fd030a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_af96c243-6c6a-4cb4-8eab-8f8a25fd030a\") failed: Property not found. retry in 713.398326ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:18.000662648Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_af96c243-6c6a-4cb4-8eab-8f8a25fd030a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_af96c243-6c6a-4cb4-8eab-8f8a25fd030a\") failed: Property not found. retry in 642.599868ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:18.723187375Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_af96c243-6c6a-4cb4-8eab-8f8a25fd030a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_af96c243-6c6a-4cb4-8eab-8f8a25fd030a\") failed: Property not found. retry in 844.602838ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:19.376121781Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_af96c243-6c6a-4cb4-8eab-8f8a25fd030a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:20.239966124Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_ntp_af96c243-6c6a-4cb4-8eab-8f8a25fd030a"} {"msg":"Writing ledger to /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:20.240017965Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:20.240419722Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:20.240778507Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"7abd63b4-7b93-4019-ac5d-8e77e537bf89","remote_addr":"[fd00:1122:3344:101::1]:43376","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":6419566,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:20.242552926Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:626"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:20.270793676Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:20.270949499Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:58058"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_af96c243-6c6a-4cb4-8eab-8f8a25fd030a': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:20.280285258Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3554"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:20.28043171Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"5cea1b2e-7c66-433e-9dc6-28c538e700fb","remote_addr":"[fd00:1122:3344:101::1]:58058","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":9359,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:20.282652306Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:20.446849638Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:20.446946319Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:59548"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_af96c243-6c6a-4cb4-8eab-8f8a25fd030a': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:24.455018607Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3554"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:24.455125669Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"c62c72ca-bf88-4d0b-84cb-503731210eae","remote_addr":"[fd00:1122:3344:101::1]:59548","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4007998,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:24.458605164Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:24.637178166Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:24.637323368Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:60612"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_af96c243-6c6a-4cb4-8eab-8f8a25fd030a': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:24.646074838Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3554"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:24.64619387Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"0325d684-7dd9-4464-8fbe-fbf360cf8326","remote_addr":"[fd00:1122:3344:101::1]:60612","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8692,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:24.64870836Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:24.808806086Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:24.808963499Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:40051"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_af96c243-6c6a-4cb4-8eab-8f8a25fd030a': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:28.817746678Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3554"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:28.8178585Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"ae4ef90a-b111-4c0d-a659-66a076e6d39e","remote_addr":"[fd00:1122:3344:101::1]:40051","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4008744,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:28.820160047Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:28.972264855Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:28.972410977Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:47602"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:28.981412421Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"b5b4ad0a-d1e7-4307-85ef-c36a8357535e","remote_addr":"[fd00:1122:3344:101::1]:47602","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":8816,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:28.984219896Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:28.984232946Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:29.128849965Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:29.128986707Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:40238"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:29.138019892Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"7f361792-c710-4746-a275-015b20425928","remote_addr":"[fd00:1122:3344:101::1]:40238","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":8902,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:29.139618277Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:29.139623957Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:29.48048535Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:29.480676253Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:65405"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:29.491710519Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"27e7b13d-e228-4a42-82ed-2bfdb5e0b388","remote_addr":"[fd00:1122:3344:101::1]:65405","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":10867,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:29.493518538Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:29.493524238Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:29.734340953Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:423","component":"producer-server","component":"MetricsManager","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:423","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"193.171776ms"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:29.77298558Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:29.773091762Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:59066"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:29.782043275Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"c8313f3e-30be-4d3a-b618-014135a97564","remote_addr":"[fd00:1122:3344:101::1]:59066","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":8833,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:29.783472728Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:29.783478748Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:30.288033025Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:30.288166657Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:40788"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:30.296363148Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"f75a83ce-6f3a-4f8c-be8d-e2571c67dd14","remote_addr":"[fd00:1122:3344:101::1]:40788","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":8033,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:30.298023894Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:30.298033844Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:31.353960621Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:31.354106944Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:34103"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:31.36389398Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"4203880b-c122-4eff-9af5-4850e5f5cf41","remote_addr":"[fd00:1122:3344:101::1]:34103","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":9610,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:31.366977939Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:31.366988829Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:31.708600323Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:31.708713845Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:37844"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:31.718441441Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"ea5277a6-f905-46fd-aa3d-f741e4a4f7b3","remote_addr":"[fd00:1122:3344:101::1]:37844","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":9587,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:31.721401918Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:31.721412878Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:32.191330192Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:32.191447364Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:43036"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:32.200328206Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"9e29de23-6f1a-4e7e-959b-bd37db6c3a3c","remote_addr":"[fd00:1122:3344:101::1]:43036","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":8759,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:32.202610562Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:32.202617612Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:33.068146658Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:33.06827601Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:64981"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:33.077105231Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"c95f454a-e76b-40cd-a2c0-3d69ae04a562","remote_addr":"[fd00:1122:3344:101::1]:64981","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":8676,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:33.080875081Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:33.080885171Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:33.172431844Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:33.172543666Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:38827"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:33.193683993Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"4c454681-95ae-4af0-be86-7d1b6684a4e8","remote_addr":"[fd00:1122:3344:101::1]:38827","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":20994,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 882006538, ip_addr: 52.146.90.10, stratum: 2, ref_time: 1720224213.298376, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:33.195975209Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:33.19598165Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1224"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: cdec4b62-7244-4b80-8777-6d905a5202cd, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 92bc3f3a-c64a-4323-b745-aba7d1b54a8d, kind: Service { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a }, name: Name(\"ntp-af96c243-6c6a-4cb4-8eab-8f8a25fd030a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 160])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f85a133-0257-47cf-8244-149e630b6fde, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: e38bc018-21d0-4daa-8b2c-d199004af618, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 69cbd71e-481b-406f-9eb3-20fa7a960393, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:33.224144219Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:33.224263991Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:48222"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:33.224495395Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:33.224505445Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:33.245658273Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:33.245706874Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:33.266030058Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:33.266083339Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:33.284823858Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:33.284862079Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:33.337497539Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:33.33754482Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:33.391939629Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:33.391982499Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:33.446314647Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:33.446353327Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:33.498915207Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:33.498961848Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:33.554655097Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:33.554715878Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:33.554942781Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:33.555042953Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:33.640152282Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:33.640242514Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:33.640248564Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:33.640255874Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:33.640260954Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_69cbd71e-481b-406f-9eb3-20fa7a960393","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:33.728170178Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_e38bc018-21d0-4daa-8b2c-d199004af618","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:33.732597388Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_4f5c13e8-8c8d-45df-9213-4b813bd44490","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:33.736524811Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_3f85a133-0257-47cf-8244-149e630b6fde","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:33.740053977Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_375efe4d-2c7a-4598-9c88-6c4c60fbd4a4","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:33.74462439Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_cockroachdb_69cbd71e-481b-406f-9eb3-20fa7a960393","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:33.749427857Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_e38bc018-21d0-4daa-8b2c-d199004af618","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:33.754691781Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_4f5c13e8-8c8d-45df-9213-4b813bd44490","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:33.759358865Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_3f85a133-0257-47cf-8244-149e630b6fde","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:33.763778636Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_375efe4d-2c7a-4598-9c88-6c4c60fbd4a4","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:33.768092475Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_cockroachdb_e38bc018-21d0-4daa-8b2c-d199004af618:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:36.737776432Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:36.737905844Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_e38bc018-21d0-4daa-8b2c-d199004af618","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_3f85a133-0257-47cf-8244-149e630b6fde:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:36.753216869Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:36.753352581Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3f85a133-0257-47cf-8244-149e630b6fde","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_69cbd71e-481b-406f-9eb3-20fa7a960393:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:36.766639413Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:36.766779895Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_69cbd71e-481b-406f-9eb3-20fa7a960393","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_4f5c13e8-8c8d-45df-9213-4b813bd44490:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:36.781608182Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:36.781762044Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_4f5c13e8-8c8d-45df-9213-4b813bd44490","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_e38bc018-21d0-4daa-8b2c-d199004af618\") failed: Property not found. retry in 50.073699ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:36.8866906Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_e38bc018-21d0-4daa-8b2c-d199004af618","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_cockroachdb_375efe4d-2c7a-4598-9c88-6c4c60fbd4a4:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:36.889931132Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:36.890065264Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_375efe4d-2c7a-4598-9c88-6c4c60fbd4a4","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_3f85a133-0257-47cf-8244-149e630b6fde\") failed: Property not found. retry in 73.835215ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:36.903136082Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3f85a133-0257-47cf-8244-149e630b6fde","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_69cbd71e-481b-406f-9eb3-20fa7a960393\") failed: Property not found. retry in 58.428527ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:36.910821075Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_69cbd71e-481b-406f-9eb3-20fa7a960393","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4f5c13e8-8c8d-45df-9213-4b813bd44490\") failed: Property not found. retry in 38.798485ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:36.922662644Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_4f5c13e8-8c8d-45df-9213-4b813bd44490","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e38bc018-21d0-4daa-8b2c-d199004af618\") failed: Property not found. retry in 78.309408ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:36.947022293Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_e38bc018-21d0-4daa-8b2c-d199004af618","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4f5c13e8-8c8d-45df-9213-4b813bd44490\") failed: Property not found. retry in 58.566816ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:37.002047682Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_4f5c13e8-8c8d-45df-9213-4b813bd44490","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_69cbd71e-481b-406f-9eb3-20fa7a960393\") failed: Property not found. retry in 122.826102ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:37.113071725Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_69cbd71e-481b-406f-9eb3-20fa7a960393","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3f85a133-0257-47cf-8244-149e630b6fde\") failed: Property not found. retry in 107.08567ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:37.313370443Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3f85a133-0257-47cf-8244-149e630b6fde","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_69cbd71e-481b-406f-9eb3-20fa7a960393\") failed: Property not found. retry in 257.856889ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:37.64760111Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_69cbd71e-481b-406f-9eb3-20fa7a960393","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e38bc018-21d0-4daa-8b2c-d199004af618\") failed: Property not found. retry in 114.996627ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:37.776024961Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_e38bc018-21d0-4daa-8b2c-d199004af618","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4f5c13e8-8c8d-45df-9213-4b813bd44490\") failed: Property not found. retry in 207.84206ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:37.914266247Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_4f5c13e8-8c8d-45df-9213-4b813bd44490","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3f85a133-0257-47cf-8244-149e630b6fde\") failed: Property not found. retry in 143.291703ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:38.093250945Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3f85a133-0257-47cf-8244-149e630b6fde","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_375efe4d-2c7a-4598-9c88-6c4c60fbd4a4\") failed: Property not found. retry in 65.114714ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:38.223721279Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_375efe4d-2c7a-4598-9c88-6c4c60fbd4a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_69cbd71e-481b-406f-9eb3-20fa7a960393\") failed: Property not found. retry in 261.005591ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:38.353858677Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_69cbd71e-481b-406f-9eb3-20fa7a960393","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e38bc018-21d0-4daa-8b2c-d199004af618\") failed: Property not found. retry in 432.575078ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:38.627921183Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_e38bc018-21d0-4daa-8b2c-d199004af618","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4f5c13e8-8c8d-45df-9213-4b813bd44490\") failed: Property not found. retry in 501.540482ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:38.748909935Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_4f5c13e8-8c8d-45df-9213-4b813bd44490","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3f85a133-0257-47cf-8244-149e630b6fde\") failed: Property not found. retry in 596.855307ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:38.819123936Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3f85a133-0257-47cf-8244-149e630b6fde","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_375efe4d-2c7a-4598-9c88-6c4c60fbd4a4\") failed: Property not found. retry in 112.254268ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:39.027781538Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_375efe4d-2c7a-4598-9c88-6c4c60fbd4a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_69cbd71e-481b-406f-9eb3-20fa7a960393\") failed: Property not found. retry in 783.443985ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:39.163974313Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_69cbd71e-481b-406f-9eb3-20fa7a960393","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e38bc018-21d0-4daa-8b2c-d199004af618\") failed: Property not found. retry in 895.398062ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:39.209225675Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_e38bc018-21d0-4daa-8b2c-d199004af618","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_375efe4d-2c7a-4598-9c88-6c4c60fbd4a4\") failed: Property not found. retry in 131.315365ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:39.321598379Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_375efe4d-2c7a-4598-9c88-6c4c60fbd4a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4f5c13e8-8c8d-45df-9213-4b813bd44490\") failed: Property not found. retry in 1.050774138s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:39.36106975Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_4f5c13e8-8c8d-45df-9213-4b813bd44490","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3f85a133-0257-47cf-8244-149e630b6fde\") failed: Property not found. retry in 1.095996685s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:39.479205446Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3f85a133-0257-47cf-8244-149e630b6fde","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_375efe4d-2c7a-4598-9c88-6c4c60fbd4a4\") failed: Property not found. retry in 528.449022ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:39.554503509Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_375efe4d-2c7a-4598-9c88-6c4c60fbd4a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_69cbd71e-481b-406f-9eb3-20fa7a960393\") failed: Property not found. retry in 827.443008ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:39.959602426Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_69cbd71e-481b-406f-9eb3-20fa7a960393","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_375efe4d-2c7a-4598-9c88-6c4c60fbd4a4\") failed: Property not found. retry in 961.554879ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:40.094394278Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_375efe4d-2c7a-4598-9c88-6c4c60fbd4a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e38bc018-21d0-4daa-8b2c-d199004af618\") failed: Property not found. retry in 569.652021ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:40.118793138Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_e38bc018-21d0-4daa-8b2c-d199004af618","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4f5c13e8-8c8d-45df-9213-4b813bd44490\") failed: Property not found. retry in 1.485476268s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:40.419708323Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_4f5c13e8-8c8d-45df-9213-4b813bd44490","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3f85a133-0257-47cf-8244-149e630b6fde\") failed: Property not found. retry in 1.315569972s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:40.583916955Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3f85a133-0257-47cf-8244-149e630b6fde","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e38bc018-21d0-4daa-8b2c-d199004af618\") failed: Property not found. retry in 614.128574ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:40.695595038Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_e38bc018-21d0-4daa-8b2c-d199004af618","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_69cbd71e-481b-406f-9eb3-20fa7a960393\") failed: Property not found. retry in 955.543238ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:40.79401788Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_69cbd71e-481b-406f-9eb3-20fa7a960393","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_375efe4d-2c7a-4598-9c88-6c4c60fbd4a4\") failed: Property not found. retry in 1.23141705s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:41.065321621Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_375efe4d-2c7a-4598-9c88-6c4c60fbd4a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e38bc018-21d0-4daa-8b2c-d199004af618\") failed: Property not found. retry in 741.829147ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:41.318610776Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_e38bc018-21d0-4daa-8b2c-d199004af618","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_69cbd71e-481b-406f-9eb3-20fa7a960393\") failed: Property not found. retry in 1.207158449s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:41.759253422Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_69cbd71e-481b-406f-9eb3-20fa7a960393","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3f85a133-0257-47cf-8244-149e630b6fde\") failed: Property not found. retry in 586.023829ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:41.910418865Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3f85a133-0257-47cf-8244-149e630b6fde","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4f5c13e8-8c8d-45df-9213-4b813bd44490\") failed: Property not found. retry in 814.988604ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:41.917737162Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_4f5c13e8-8c8d-45df-9213-4b813bd44490","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e38bc018-21d0-4daa-8b2c-d199004af618\") failed: Property not found. retry in 1.061945368s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:42.068881765Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_e38bc018-21d0-4daa-8b2c-d199004af618","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_375efe4d-2c7a-4598-9c88-6c4c60fbd4a4\") failed: Property not found. retry in 1.368925872s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:42.305986191Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_375efe4d-2c7a-4598-9c88-6c4c60fbd4a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3f85a133-0257-47cf-8244-149e630b6fde\") failed: Property not found. retry in 707.704724ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:42.504967879Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3f85a133-0257-47cf-8244-149e630b6fde","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4f5c13e8-8c8d-45df-9213-4b813bd44490\") failed: Property not found. retry in 890.380344ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:42.741829501Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_4f5c13e8-8c8d-45df-9213-4b813bd44490","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_69cbd71e-481b-406f-9eb3-20fa7a960393\") failed: Property not found. retry in 1.248604894s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:42.975673614Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_69cbd71e-481b-406f-9eb3-20fa7a960393","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e38bc018-21d0-4daa-8b2c-d199004af618\") failed: Property not found. retry in 1.318386957s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:43.139112533Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_e38bc018-21d0-4daa-8b2c-d199004af618","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3f85a133-0257-47cf-8244-149e630b6fde\") failed: Property not found. retry in 1.292012965s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:43.221148434Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3f85a133-0257-47cf-8244-149e630b6fde","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4f5c13e8-8c8d-45df-9213-4b813bd44490\") failed: Property not found. retry in 607.141001ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:43.641864121Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_4f5c13e8-8c8d-45df-9213-4b813bd44490","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_375efe4d-2c7a-4598-9c88-6c4c60fbd4a4\") failed: Property not found. retry in 1.47445673s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:43.683824931Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_375efe4d-2c7a-4598-9c88-6c4c60fbd4a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_69cbd71e-481b-406f-9eb3-20fa7a960393\") failed: Property not found. retry in 945.616817ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:44.233072301Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_69cbd71e-481b-406f-9eb3-20fa7a960393","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4f5c13e8-8c8d-45df-9213-4b813bd44490\") failed: Property not found. retry in 681.7438ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:44.259056856Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_4f5c13e8-8c8d-45df-9213-4b813bd44490","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e38bc018-21d0-4daa-8b2c-d199004af618\") failed: Property not found. retry in 623.373736ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:44.466378686Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_e38bc018-21d0-4daa-8b2c-d199004af618","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3f85a133-0257-47cf-8244-149e630b6fde\") failed: Property not found. retry in 953.2503ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:44.522142577Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3f85a133-0257-47cf-8244-149e630b6fde","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:44.896132788Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:423","component":"producer-server","component":"MetricsManager","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:423","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"545.614809ms"} {"msg":"wait for service Some(\"oxz_cockroachdb_4f5c13e8-8c8d-45df-9213-4b813bd44490\") failed: Property not found. retry in 1.275525415s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:44.950551277Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_4f5c13e8-8c8d-45df-9213-4b813bd44490","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e38bc018-21d0-4daa-8b2c-d199004af618\") failed: Property not found. retry in 548.018165ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:45.099320012Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_e38bc018-21d0-4daa-8b2c-d199004af618","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_375efe4d-2c7a-4598-9c88-6c4c60fbd4a4\") failed: Property not found. retry in 788.799139ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:45.166244641Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_375efe4d-2c7a-4598-9c88-6c4c60fbd4a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_69cbd71e-481b-406f-9eb3-20fa7a960393\") failed: Property not found. retry in 974.976632ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:45.187705103Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_69cbd71e-481b-406f-9eb3-20fa7a960393","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3f85a133-0257-47cf-8244-149e630b6fde\") failed: Property not found. retry in 1.154454794s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:45.483657149Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3f85a133-0257-47cf-8244-149e630b6fde","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e38bc018-21d0-4daa-8b2c-d199004af618\") failed: Property not found. retry in 864.408442ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:45.656490169Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_e38bc018-21d0-4daa-8b2c-d199004af618","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_375efe4d-2c7a-4598-9c88-6c4c60fbd4a4\") failed: Property not found. retry in 1.496032939s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:45.963822026Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_375efe4d-2c7a-4598-9c88-6c4c60fbd4a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_69cbd71e-481b-406f-9eb3-20fa7a960393\") failed: Property not found. retry in 616.692117ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:46.172764422Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_69cbd71e-481b-406f-9eb3-20fa7a960393","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4f5c13e8-8c8d-45df-9213-4b813bd44490\") failed: Property not found. retry in 1.395905382s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:46.23710449Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_4f5c13e8-8c8d-45df-9213-4b813bd44490","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e38bc018-21d0-4daa-8b2c-d199004af618\") failed: Property not found. retry in 1.283826991s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:46.532265403Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_e38bc018-21d0-4daa-8b2c-d199004af618","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:46.62227366Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:50693"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:46.622416282Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"cbff0223-3ddc-489e-8d86-878b63dd5cc0","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:50693","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":85,"response_code":"200"} {"msg":"wait for service Some(\"oxz_cockroachdb_3f85a133-0257-47cf-8244-149e630b6fde\") failed: Property not found. retry in 1.058016487s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:46.650949048Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3f85a133-0257-47cf-8244-149e630b6fde","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_69cbd71e-481b-406f-9eb3-20fa7a960393\") failed: Property not found. retry in 1.207674421s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:46.799277836Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_69cbd71e-481b-406f-9eb3-20fa7a960393","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_375efe4d-2c7a-4598-9c88-6c4c60fbd4a4\") failed: Property not found. retry in 1.151544555s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:47.469994585Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_375efe4d-2c7a-4598-9c88-6c4c60fbd4a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4f5c13e8-8c8d-45df-9213-4b813bd44490\") failed: Property not found. retry in 1.442613973s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:47.642280526Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_4f5c13e8-8c8d-45df-9213-4b813bd44490","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3f85a133-0257-47cf-8244-149e630b6fde\") failed: Property not found. retry in 838.265062ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:47.717494407Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3f85a133-0257-47cf-8244-149e630b6fde","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e38bc018-21d0-4daa-8b2c-d199004af618\") failed: Property not found. retry in 841.703249ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:47.824210801Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_e38bc018-21d0-4daa-8b2c-d199004af618","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_69cbd71e-481b-406f-9eb3-20fa7a960393\") failed: Property not found. retry in 542.813402ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:48.016129495Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_69cbd71e-481b-406f-9eb3-20fa7a960393","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3f85a133-0257-47cf-8244-149e630b6fde\") failed: Property not found. retry in 786.051777ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:48.565275614Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3f85a133-0257-47cf-8244-149e630b6fde","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_69cbd71e-481b-406f-9eb3-20fa7a960393\") failed: Property not found. retry in 597.261778ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:48.572861085Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_69cbd71e-481b-406f-9eb3-20fa7a960393","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_375efe4d-2c7a-4598-9c88-6c4c60fbd4a4\") failed: Property not found. retry in 765.761674ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:48.630261761Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_375efe4d-2c7a-4598-9c88-6c4c60fbd4a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e38bc018-21d0-4daa-8b2c-d199004af618\") failed: Property not found. retry in 963.110147ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:48.674747832Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_e38bc018-21d0-4daa-8b2c-d199004af618","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4f5c13e8-8c8d-45df-9213-4b813bd44490\") failed: Property not found. retry in 829.191031ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:49.094169698Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_4f5c13e8-8c8d-45df-9213-4b813bd44490","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_69cbd71e-481b-406f-9eb3-20fa7a960393\") failed: Property not found. retry in 1.062125152s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:49.178937262Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_69cbd71e-481b-406f-9eb3-20fa7a960393","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3f85a133-0257-47cf-8244-149e630b6fde\") failed: Property not found. retry in 1.128709131s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:49.360725334Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3f85a133-0257-47cf-8244-149e630b6fde","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_375efe4d-2c7a-4598-9c88-6c4c60fbd4a4\") failed: Property not found. retry in 793.882217ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:49.403982325Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_375efe4d-2c7a-4598-9c88-6c4c60fbd4a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e38bc018-21d0-4daa-8b2c-d199004af618\") failed: Property not found. retry in 593.688287ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:49.648744193Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_e38bc018-21d0-4daa-8b2c-d199004af618","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4f5c13e8-8c8d-45df-9213-4b813bd44490\") failed: Property not found. retry in 894.170498ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:49.932579006Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_4f5c13e8-8c8d-45df-9213-4b813bd44490","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_375efe4d-2c7a-4598-9c88-6c4c60fbd4a4\") failed: Property not found. retry in 600.883379ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:50.20779997Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_375efe4d-2c7a-4598-9c88-6c4c60fbd4a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_69cbd71e-481b-406f-9eb3-20fa7a960393\") failed: Property not found. retry in 888.573125ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:50.250082606Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_69cbd71e-481b-406f-9eb3-20fa7a960393","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e38bc018-21d0-4daa-8b2c-d199004af618\") failed: Property not found. retry in 577.369822ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:50.257764438Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_e38bc018-21d0-4daa-8b2c-d199004af618","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:51.162670207Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_cockroachdb_69cbd71e-481b-406f-9eb3-20fa7a960393"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:51.162754338Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_cockroachdb_e38bc018-21d0-4daa-8b2c-d199004af618"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:51.162761538Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_cockroachdb_4f5c13e8-8c8d-45df-9213-4b813bd44490"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:51.162767548Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_cockroachdb_3f85a133-0257-47cf-8244-149e630b6fde"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:51.162774928Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_cockroachdb_375efe4d-2c7a-4598-9c88-6c4c60fbd4a4"} {"msg":"Writing ledger to /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:51.162793969Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:51.163222446Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:51.163530391Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"453e2848-9d2a-454b-a792-fab264028f48","remote_addr":"[fd00:1122:3344:101::1]:48222","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":17939208,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:51.195244197Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:38596"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:51.228696081Z","hostname":"sock","pid":1555,"uri":"/cockroachdb","method":"POST","req_id":"be077aae-3b12-4c21-b011-efa6c66d7eb0","remote_addr":"[fd00:1122:3344:101::1]:38596","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService9/omicron6 in oxz_cockroachdb_375efe4d-2c7a-4598-9c88-6c4c60fbd4a4: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_375efe4d-2c7a-4598-9c88-6c4c60fbd4a4 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService9/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":33302,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:51.229001676Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:966","retry_after":"106.731497ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"be077aae-3b12-4c21-b011-efa6c66d7eb0\", \"content-length\": \"124\", \"date\": \"Sat, 06 Jul 2024 00:03:51 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"be077aae-3b12-4c21-b011-efa6c66d7eb0\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:51.37078774Z","hostname":"sock","pid":1555,"uri":"/cockroachdb","method":"POST","req_id":"6df38cb2-a793-4ab3-9491-6db047e97126","remote_addr":"[fd00:1122:3344:101::1]:38596","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService9/omicron6 in oxz_cockroachdb_375efe4d-2c7a-4598-9c88-6c4c60fbd4a4: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_375efe4d-2c7a-4598-9c88-6c4c60fbd4a4 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService9/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":33495,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:51.370995723Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:966","retry_after":"161.050583ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6df38cb2-a793-4ab3-9491-6db047e97126\", \"content-length\": \"124\", \"date\": \"Sat, 06 Jul 2024 00:03:51 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6df38cb2-a793-4ab3-9491-6db047e97126\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:51.566324262Z","hostname":"sock","pid":1555,"uri":"/cockroachdb","method":"POST","req_id":"03056e0a-a8ce-4c32-9d82-281e5848e80c","remote_addr":"[fd00:1122:3344:101::1]:38596","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService9/omicron6 in oxz_cockroachdb_375efe4d-2c7a-4598-9c88-6c4c60fbd4a4: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_375efe4d-2c7a-4598-9c88-6c4c60fbd4a4 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService9/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":33031,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:51.566518225Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:966","retry_after":"162.735794ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"03056e0a-a8ce-4c32-9d82-281e5848e80c\", \"content-length\": \"124\", \"date\": \"Sat, 06 Jul 2024 00:03:51 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"03056e0a-a8ce-4c32-9d82-281e5848e80c\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:51.757688048Z","hostname":"sock","pid":1555,"uri":"/cockroachdb","method":"POST","req_id":"352061f7-e569-4b15-8014-93631af83f68","remote_addr":"[fd00:1122:3344:101::1]:38596","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService9/omicron6 in oxz_cockroachdb_375efe4d-2c7a-4598-9c88-6c4c60fbd4a4: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_375efe4d-2c7a-4598-9c88-6c4c60fbd4a4 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService9/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":27835,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:03:51.75785123Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:966","retry_after":"190.574886ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"352061f7-e569-4b15-8014-93631af83f68\", \"content-length\": \"124\", \"date\": \"Sat, 06 Jul 2024 00:03:51 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"352061f7-e569-4b15-8014-93631af83f68\" }"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::7]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:03:51.982327164Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3388"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:14.003264867Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3406"} {"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:22.867070948Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3429"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:22.867515165Z","hostname":"sock","pid":1555,"uri":"/cockroachdb","method":"POST","req_id":"51f40029-78c3-4364-bec7-7f07c42fa4c3","remote_addr":"[fd00:1122:3344:101::1]:38596","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":30918411,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:22.867774769Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:54186"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:22.868549421Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"e28faf87-5bac-45f6-9de6-ac1410abd0b0","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:54186","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":177,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:22.873652193Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:423","component":"producer-server","component":"MetricsManager","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:423","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"1.059511872s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: cdec4b62-7244-4b80-8777-6d905a5202cd, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 92bc3f3a-c64a-4323-b745-aba7d1b54a8d, kind: Service { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a }, name: Name(\"ntp-af96c243-6c6a-4cb4-8eab-8f8a25fd030a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 160])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f85a133-0257-47cf-8244-149e630b6fde, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: e38bc018-21d0-4daa-8b2c-d199004af618, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 69cbd71e-481b-406f-9eb3-20fa7a960393, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 544a0264-7c3a-4ada-aa6e-553800c85da2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 4d9df03f-9907-4449-8d22-3a38f32b899c, kind: Service { id: 544a0264-7c3a-4ada-aa6e-553800c85da2 }, name: Name(\"external-dns-544a0264-7c3a-4ada-aa6e-553800c85da2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 114])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: f0ddfaba-8652-4628-848a-f993e079fb94, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 0ab9fb64-2e84-44e1-91ff-d472c798d3d1, kind: Service { id: f0ddfaba-8652-4628-848a-f993e079fb94 }, name: Name(\"external-dns-f0ddfaba-8652-4628-848a-f993e079fb94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 96bac47d-0e2f-4191-820f-fdc11f7f9342, kind: Service { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f }, name: Name(\"nexus-9ff5ab27-a3ab-4401-8a23-c0799de85f2f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e46afa6-9da3-407c-92bf-128b95063f04, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4ad4ab3a-b527-470c-b61b-a8987f3919b5, kind: Service { id: 8e46afa6-9da3-407c-92bf-128b95063f04 }, name: Name(\"nexus-8e46afa6-9da3-407c-92bf-128b95063f04\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ab5e3564-209c-48fa-b8f5-aa370048ac8d, kind: Service { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67 }, name: Name(\"nexus-7b52f14b-12cb-45e4-b4e3-5c517e3cba67\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 39])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: da35a429-5291-41f8-8c2f-7377bb15ed38, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5346055-3249-4357-951e-e5bf8d4dc3cf, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: be250b1e-18db-457d-9646-485f60896fb8, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f545bea-eb7a-44c2-a399-0237abf2a102, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e35814c6-9e18-4226-86a8-2ab44a0293b2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 82bbc823-811c-4ea4-adbb-453cfe570acf, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42568398-8d6b-43e8-b33e-d40f8662e422, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External } } } }, OmicronZoneConfig { id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41863173-79c0-4807-9ede-f5bf31ee3536, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68e6a369-307f-4afa-a667-cfb114baa982, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:22.905876407Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:22.906064019Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:35174"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:22.907383971Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:22.907391511Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:22.931520656Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:22.931563677Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:22.951953992Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:22.952003583Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:22.971402063Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:22.971467514Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:22.99124235Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:22.99129039Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:23.009260017Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:23.009306368Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:23.028782529Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:23.02882413Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:23.047225664Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:23.047267094Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:23.068603805Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:23.068648026Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:23.15724126Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:23.157292671Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:23.246425434Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:23.246479635Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:23.326445062Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:23.326499133Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:23.408656695Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:23.408714996Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:23.489041778Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:23.489101599Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:23.56990437Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:23.5699585Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:23.651681025Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:23.651736776Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:23.730086027Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:23.730139128Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:23.814930772Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:23.814990443Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:23.906400232Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:23.906458933Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:23.992884503Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:23.992923944Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.074585397Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.074644778Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.074988424Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.075179937Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:24.190277665Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:24.190338166Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:24.190345936Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:24.190376196Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:24.190383526Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:24.190390386Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:24.190395937Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:24.190403577Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:24.190409017Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:24.190414527Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:24.190421387Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:24.190426807Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:24.190432367Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:24.190437567Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:24.190444667Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:24.190449987Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:24.190456908Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:24.190462308Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:24.190467878Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.190567139Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"91deb55d-4224-48da-a092-c6cbbed57e90","remote_addr":"[fd00:1122:3344:101::1]:35174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\", TimeNotSynchronized), (\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\", TimeNotSynchronized), (\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\", TimeNotSynchronized), (\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\", TimeNotSynchronized), (\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\", TimeNotSynchronized), (\"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67\", TimeNotSynchronized), (\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\", TimeNotSynchronized), (\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\", TimeNotSynchronized), (\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\", TimeNotSynchronized), (\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\", TimeNotSynchronized), (\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\", TimeNotSynchronized), (\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\", TimeNotSynchronized), (\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\", TimeNotSynchronized), (\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\", TimeNotSynchronized), (\"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\", TimeNotSynchronized), (\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\", TimeNotSynchronized), (\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\", TimeNotSynchronized), (\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\", TimeNotSynchronized)]","latency_us":1284005,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:24.190843984Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"130.315202ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"91deb55d-4224-48da-a092-c6cbbed57e90\", \"content-length\": \"124\", \"date\": \"Sat, 06 Jul 2024 00:04:24 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"91deb55d-4224-48da-a092-c6cbbed57e90\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: cdec4b62-7244-4b80-8777-6d905a5202cd, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 92bc3f3a-c64a-4323-b745-aba7d1b54a8d, kind: Service { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a }, name: Name(\"ntp-af96c243-6c6a-4cb4-8eab-8f8a25fd030a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 160])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f85a133-0257-47cf-8244-149e630b6fde, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: e38bc018-21d0-4daa-8b2c-d199004af618, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 69cbd71e-481b-406f-9eb3-20fa7a960393, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 544a0264-7c3a-4ada-aa6e-553800c85da2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 4d9df03f-9907-4449-8d22-3a38f32b899c, kind: Service { id: 544a0264-7c3a-4ada-aa6e-553800c85da2 }, name: Name(\"external-dns-544a0264-7c3a-4ada-aa6e-553800c85da2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 114])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: f0ddfaba-8652-4628-848a-f993e079fb94, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 0ab9fb64-2e84-44e1-91ff-d472c798d3d1, kind: Service { id: f0ddfaba-8652-4628-848a-f993e079fb94 }, name: Name(\"external-dns-f0ddfaba-8652-4628-848a-f993e079fb94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 96bac47d-0e2f-4191-820f-fdc11f7f9342, kind: Service { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f }, name: Name(\"nexus-9ff5ab27-a3ab-4401-8a23-c0799de85f2f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e46afa6-9da3-407c-92bf-128b95063f04, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4ad4ab3a-b527-470c-b61b-a8987f3919b5, kind: Service { id: 8e46afa6-9da3-407c-92bf-128b95063f04 }, name: Name(\"nexus-8e46afa6-9da3-407c-92bf-128b95063f04\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ab5e3564-209c-48fa-b8f5-aa370048ac8d, kind: Service { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67 }, name: Name(\"nexus-7b52f14b-12cb-45e4-b4e3-5c517e3cba67\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 39])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: da35a429-5291-41f8-8c2f-7377bb15ed38, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5346055-3249-4357-951e-e5bf8d4dc3cf, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: be250b1e-18db-457d-9646-485f60896fb8, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f545bea-eb7a-44c2-a399-0237abf2a102, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e35814c6-9e18-4226-86a8-2ab44a0293b2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 82bbc823-811c-4ea4-adbb-453cfe570acf, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42568398-8d6b-43e8-b33e-d40f8662e422, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External } } } }, OmicronZoneConfig { id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41863173-79c0-4807-9ede-f5bf31ee3536, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68e6a369-307f-4afa-a667-cfb114baa982, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.322025459Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.323286349Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.323295699Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.346776824Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.346841025Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.368618212Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.368668603Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.388280617Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.388329387Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.407436432Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.407479893Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.428444008Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.428498089Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.447525272Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.447568763Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.466398064Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.466446554Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.486475864Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.486524165Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.505642671Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.505698421Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.524856747Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.524901108Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.545625139Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.54566924Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.564059293Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.564110694Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.582485558Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.582541958Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.601835307Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.601884317Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.622623358Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.622673449Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.642866652Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.642911552Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.663543972Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.663595403Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.685923859Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.68596627Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.706983195Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.707033376Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.726472147Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.726539838Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.726759641Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.726906944Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:24.826578335Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:24.826645236Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:24.826654336Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:24.826661837Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:24.826667737Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:24.826673897Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:24.826681037Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:24.826706707Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:24.826712837Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:24.826720507Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:24.826727547Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:24.826732898Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:24.826738418Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:24.826745718Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:24.826751288Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:24.826756698Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:24.826762578Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:24.826769628Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:24.826774888Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.82687026Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"be43822d-4204-42fa-8abe-804c033afd22","remote_addr":"[fd00:1122:3344:101::1]:35174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\", TimeNotSynchronized), (\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\", TimeNotSynchronized), (\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\", TimeNotSynchronized), (\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\", TimeNotSynchronized), (\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\", TimeNotSynchronized), (\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\", TimeNotSynchronized), (\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\", TimeNotSynchronized), (\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\", TimeNotSynchronized), (\"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf\", TimeNotSynchronized), (\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\", TimeNotSynchronized), (\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\", TimeNotSynchronized), (\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\", TimeNotSynchronized), (\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\", TimeNotSynchronized), (\"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67\", TimeNotSynchronized), (\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\", TimeNotSynchronized), (\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\", TimeNotSynchronized), (\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\", TimeNotSynchronized), (\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\", TimeNotSynchronized)]","latency_us":504636,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:24.827058903Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"162.948432ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"be43822d-4204-42fa-8abe-804c033afd22\", \"content-length\": \"124\", \"date\": \"Sat, 06 Jul 2024 00:04:24 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"be43822d-4204-42fa-8abe-804c033afd22\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: cdec4b62-7244-4b80-8777-6d905a5202cd, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 92bc3f3a-c64a-4323-b745-aba7d1b54a8d, kind: Service { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a }, name: Name(\"ntp-af96c243-6c6a-4cb4-8eab-8f8a25fd030a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 160])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f85a133-0257-47cf-8244-149e630b6fde, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: e38bc018-21d0-4daa-8b2c-d199004af618, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 69cbd71e-481b-406f-9eb3-20fa7a960393, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 544a0264-7c3a-4ada-aa6e-553800c85da2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 4d9df03f-9907-4449-8d22-3a38f32b899c, kind: Service { id: 544a0264-7c3a-4ada-aa6e-553800c85da2 }, name: Name(\"external-dns-544a0264-7c3a-4ada-aa6e-553800c85da2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 114])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: f0ddfaba-8652-4628-848a-f993e079fb94, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 0ab9fb64-2e84-44e1-91ff-d472c798d3d1, kind: Service { id: f0ddfaba-8652-4628-848a-f993e079fb94 }, name: Name(\"external-dns-f0ddfaba-8652-4628-848a-f993e079fb94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 96bac47d-0e2f-4191-820f-fdc11f7f9342, kind: Service { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f }, name: Name(\"nexus-9ff5ab27-a3ab-4401-8a23-c0799de85f2f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e46afa6-9da3-407c-92bf-128b95063f04, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4ad4ab3a-b527-470c-b61b-a8987f3919b5, kind: Service { id: 8e46afa6-9da3-407c-92bf-128b95063f04 }, name: Name(\"nexus-8e46afa6-9da3-407c-92bf-128b95063f04\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ab5e3564-209c-48fa-b8f5-aa370048ac8d, kind: Service { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67 }, name: Name(\"nexus-7b52f14b-12cb-45e4-b4e3-5c517e3cba67\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 39])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: da35a429-5291-41f8-8c2f-7377bb15ed38, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5346055-3249-4357-951e-e5bf8d4dc3cf, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: be250b1e-18db-457d-9646-485f60896fb8, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f545bea-eb7a-44c2-a399-0237abf2a102, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e35814c6-9e18-4226-86a8-2ab44a0293b2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 82bbc823-811c-4ea4-adbb-453cfe570acf, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42568398-8d6b-43e8-b33e-d40f8662e422, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External } } } }, OmicronZoneConfig { id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41863173-79c0-4807-9ede-f5bf31ee3536, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68e6a369-307f-4afa-a667-cfb114baa982, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.991413157Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.991899325Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:24.991907125Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.016897734Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.016959235Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.038437698Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.038485909Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.060716844Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.060765115Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.083982445Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.084030716Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.105198604Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.105241545Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.126158249Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.126207929Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.146046336Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.146093337Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.167394357Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.167446038Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.188311111Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.188357242Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.20887292Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.208934241Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.231720384Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.231767685Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.253261978Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.253310269Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.274693271Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.274748851Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.295915459Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.29595873Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.316063681Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.316122092Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.337671116Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.337715707Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.358263295Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.358309226Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.376212861Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.376261722Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.396394244Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.396436995Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.417637233Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.417691444Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.417868027Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.417984988Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:25.520225261Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:25.520284992Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:25.520293942Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:25.520299792Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:25.520305382Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:25.520311233Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:25.520319193Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:25.520324923Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:25.520330643Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:25.520336583Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:25.520342223Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:25.520349553Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:25.520372143Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:25.520378203Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:25.520384114Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:25.520389754Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:25.520396944Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:25.520401204Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:25.520406584Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.520547586Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"0532476c-e8fc-4a62-9bd5-2208a964badc","remote_addr":"[fd00:1122:3344:101::1]:35174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\", TimeNotSynchronized), (\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\", TimeNotSynchronized), (\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\", TimeNotSynchronized), (\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\", TimeNotSynchronized), (\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\", TimeNotSynchronized), (\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\", TimeNotSynchronized), (\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\", TimeNotSynchronized), (\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\", TimeNotSynchronized), (\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\", TimeNotSynchronized), (\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\", TimeNotSynchronized), (\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\", TimeNotSynchronized), (\"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67\", TimeNotSynchronized), (\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\", TimeNotSynchronized), (\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\", TimeNotSynchronized), (\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\", TimeNotSynchronized), (\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\", TimeNotSynchronized), (\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\", TimeNotSynchronized), (\"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf\", TimeNotSynchronized)]","latency_us":529005,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:25.52077933Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"88.507853ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0532476c-e8fc-4a62-9bd5-2208a964badc\", \"content-length\": \"124\", \"date\": \"Sat, 06 Jul 2024 00:04:25 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0532476c-e8fc-4a62-9bd5-2208a964badc\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: cdec4b62-7244-4b80-8777-6d905a5202cd, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 92bc3f3a-c64a-4323-b745-aba7d1b54a8d, kind: Service { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a }, name: Name(\"ntp-af96c243-6c6a-4cb4-8eab-8f8a25fd030a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 160])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f85a133-0257-47cf-8244-149e630b6fde, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: e38bc018-21d0-4daa-8b2c-d199004af618, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 69cbd71e-481b-406f-9eb3-20fa7a960393, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 544a0264-7c3a-4ada-aa6e-553800c85da2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 4d9df03f-9907-4449-8d22-3a38f32b899c, kind: Service { id: 544a0264-7c3a-4ada-aa6e-553800c85da2 }, name: Name(\"external-dns-544a0264-7c3a-4ada-aa6e-553800c85da2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 114])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: f0ddfaba-8652-4628-848a-f993e079fb94, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 0ab9fb64-2e84-44e1-91ff-d472c798d3d1, kind: Service { id: f0ddfaba-8652-4628-848a-f993e079fb94 }, name: Name(\"external-dns-f0ddfaba-8652-4628-848a-f993e079fb94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 96bac47d-0e2f-4191-820f-fdc11f7f9342, kind: Service { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f }, name: Name(\"nexus-9ff5ab27-a3ab-4401-8a23-c0799de85f2f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e46afa6-9da3-407c-92bf-128b95063f04, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4ad4ab3a-b527-470c-b61b-a8987f3919b5, kind: Service { id: 8e46afa6-9da3-407c-92bf-128b95063f04 }, name: Name(\"nexus-8e46afa6-9da3-407c-92bf-128b95063f04\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ab5e3564-209c-48fa-b8f5-aa370048ac8d, kind: Service { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67 }, name: Name(\"nexus-7b52f14b-12cb-45e4-b4e3-5c517e3cba67\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 39])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: da35a429-5291-41f8-8c2f-7377bb15ed38, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5346055-3249-4357-951e-e5bf8d4dc3cf, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: be250b1e-18db-457d-9646-485f60896fb8, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f545bea-eb7a-44c2-a399-0237abf2a102, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e35814c6-9e18-4226-86a8-2ab44a0293b2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 82bbc823-811c-4ea4-adbb-453cfe570acf, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42568398-8d6b-43e8-b33e-d40f8662e422, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External } } } }, OmicronZoneConfig { id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41863173-79c0-4807-9ede-f5bf31ee3536, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68e6a369-307f-4afa-a667-cfb114baa982, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.610197918Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.610561063Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.610569834Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.635926939Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.635979009Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.658867645Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.658931226Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.683077932Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.683141022Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.705119064Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.705170554Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.726638977Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.726695618Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.749765056Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.749823047Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.772987927Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.773072678Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.795991375Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.796044505Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.817869364Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.817922115Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.839332167Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.839383957Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.861585842Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.861639723Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.882646898Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.882697789Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.902747548Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.902803549Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.924360864Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.924406004Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.945812716Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.945857447Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.967636184Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.967687095Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.98990522Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:25.989957861Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.011028827Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.011086268Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.032249846Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.032302347Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.055736541Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.055801282Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.056032006Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.056184958Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:26.159720552Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:26.159792033Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:26.159802343Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:26.159808613Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:26.159814453Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:26.159821093Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:26.159826693Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:26.159831644Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:26.159837324Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:26.159843894Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:26.159849114Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:26.159853004Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:26.159856754Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:26.159862224Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:26.159866064Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:26.159869824Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:26.159888574Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:26.159892804Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:26.159898265Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.159983936Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"c025d984-9350-4260-9634-0f67a316b9c2","remote_addr":"[fd00:1122:3344:101::1]:35174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf\", TimeNotSynchronized), (\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\", TimeNotSynchronized), (\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\", TimeNotSynchronized), (\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\", TimeNotSynchronized), (\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\", TimeNotSynchronized), (\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\", TimeNotSynchronized), (\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\", TimeNotSynchronized), (\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\", TimeNotSynchronized), (\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\", TimeNotSynchronized), (\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\", TimeNotSynchronized), (\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\", TimeNotSynchronized), (\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\", TimeNotSynchronized), (\"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67\", TimeNotSynchronized), (\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\", TimeNotSynchronized), (\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\", TimeNotSynchronized), (\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\", TimeNotSynchronized), (\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\", TimeNotSynchronized), (\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\", TimeNotSynchronized)]","latency_us":549604,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:26.160182779Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"94.861099ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c025d984-9350-4260-9634-0f67a316b9c2\", \"content-length\": \"124\", \"date\": \"Sat, 06 Jul 2024 00:04:25 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c025d984-9350-4260-9634-0f67a316b9c2\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: cdec4b62-7244-4b80-8777-6d905a5202cd, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 92bc3f3a-c64a-4323-b745-aba7d1b54a8d, kind: Service { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a }, name: Name(\"ntp-af96c243-6c6a-4cb4-8eab-8f8a25fd030a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 160])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f85a133-0257-47cf-8244-149e630b6fde, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: e38bc018-21d0-4daa-8b2c-d199004af618, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 69cbd71e-481b-406f-9eb3-20fa7a960393, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 544a0264-7c3a-4ada-aa6e-553800c85da2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 4d9df03f-9907-4449-8d22-3a38f32b899c, kind: Service { id: 544a0264-7c3a-4ada-aa6e-553800c85da2 }, name: Name(\"external-dns-544a0264-7c3a-4ada-aa6e-553800c85da2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 114])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: f0ddfaba-8652-4628-848a-f993e079fb94, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 0ab9fb64-2e84-44e1-91ff-d472c798d3d1, kind: Service { id: f0ddfaba-8652-4628-848a-f993e079fb94 }, name: Name(\"external-dns-f0ddfaba-8652-4628-848a-f993e079fb94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 96bac47d-0e2f-4191-820f-fdc11f7f9342, kind: Service { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f }, name: Name(\"nexus-9ff5ab27-a3ab-4401-8a23-c0799de85f2f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e46afa6-9da3-407c-92bf-128b95063f04, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4ad4ab3a-b527-470c-b61b-a8987f3919b5, kind: Service { id: 8e46afa6-9da3-407c-92bf-128b95063f04 }, name: Name(\"nexus-8e46afa6-9da3-407c-92bf-128b95063f04\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ab5e3564-209c-48fa-b8f5-aa370048ac8d, kind: Service { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67 }, name: Name(\"nexus-7b52f14b-12cb-45e4-b4e3-5c517e3cba67\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 39])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: da35a429-5291-41f8-8c2f-7377bb15ed38, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5346055-3249-4357-951e-e5bf8d4dc3cf, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: be250b1e-18db-457d-9646-485f60896fb8, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f545bea-eb7a-44c2-a399-0237abf2a102, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e35814c6-9e18-4226-86a8-2ab44a0293b2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 82bbc823-811c-4ea4-adbb-453cfe570acf, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42568398-8d6b-43e8-b33e-d40f8662e422, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External } } } }, OmicronZoneConfig { id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41863173-79c0-4807-9ede-f5bf31ee3536, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68e6a369-307f-4afa-a667-cfb114baa982, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.256438556Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.256806262Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.256823232Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.27989635Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.279948042Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.300072983Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.300118613Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.320231845Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.320284515Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.340694781Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.340737662Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.360973365Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.361016906Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.380772491Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.380824002Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.400156691Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.400201222Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.420184311Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.420258262Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.441605583Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.441653224Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.461599202Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.461641003Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.48337333Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.48341873Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.502676538Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.502717519Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.521902845Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.521964036Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.541190613Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.541233674Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.562162408Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.562208629Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.581553597Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.581600778Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.601936493Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.601988284Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.621399964Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.621442324Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.640406517Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.640453618Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.660715042Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.660767032Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.660950345Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.661059967Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:26.762373965Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:26.762441396Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:26.762449686Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:26.762455716Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:26.762461306Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:26.762479186Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:26.762485127Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:26.762490896Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:26.762496377Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:26.762502087Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:26.762509297Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:26.762514937Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:26.762520477Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:26.762526017Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:26.762531697Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:26.762538817Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:26.762544117Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:26.762550008Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:26.762555908Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.76268785Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"2d508a49-10ac-4487-a61b-81dc833831ac","remote_addr":"[fd00:1122:3344:101::1]:35174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf\", TimeNotSynchronized), (\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\", TimeNotSynchronized), (\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\", TimeNotSynchronized), (\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\", TimeNotSynchronized), (\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\", TimeNotSynchronized), (\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\", TimeNotSynchronized), (\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\", TimeNotSynchronized), (\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\", TimeNotSynchronized), (\"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67\", TimeNotSynchronized), (\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\", TimeNotSynchronized), (\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\", TimeNotSynchronized), (\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\", TimeNotSynchronized), (\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\", TimeNotSynchronized), (\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\", TimeNotSynchronized), (\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\", TimeNotSynchronized), (\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\", TimeNotSynchronized), (\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\", TimeNotSynchronized), (\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\", TimeNotSynchronized)]","latency_us":506076,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:26.762939114Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"180.658482ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2d508a49-10ac-4487-a61b-81dc833831ac\", \"content-length\": \"124\", \"date\": \"Sat, 06 Jul 2024 00:04:26 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2d508a49-10ac-4487-a61b-81dc833831ac\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: cdec4b62-7244-4b80-8777-6d905a5202cd, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 92bc3f3a-c64a-4323-b745-aba7d1b54a8d, kind: Service { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a }, name: Name(\"ntp-af96c243-6c6a-4cb4-8eab-8f8a25fd030a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 160])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f85a133-0257-47cf-8244-149e630b6fde, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: e38bc018-21d0-4daa-8b2c-d199004af618, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 69cbd71e-481b-406f-9eb3-20fa7a960393, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 544a0264-7c3a-4ada-aa6e-553800c85da2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 4d9df03f-9907-4449-8d22-3a38f32b899c, kind: Service { id: 544a0264-7c3a-4ada-aa6e-553800c85da2 }, name: Name(\"external-dns-544a0264-7c3a-4ada-aa6e-553800c85da2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 114])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: f0ddfaba-8652-4628-848a-f993e079fb94, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 0ab9fb64-2e84-44e1-91ff-d472c798d3d1, kind: Service { id: f0ddfaba-8652-4628-848a-f993e079fb94 }, name: Name(\"external-dns-f0ddfaba-8652-4628-848a-f993e079fb94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 96bac47d-0e2f-4191-820f-fdc11f7f9342, kind: Service { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f }, name: Name(\"nexus-9ff5ab27-a3ab-4401-8a23-c0799de85f2f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e46afa6-9da3-407c-92bf-128b95063f04, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4ad4ab3a-b527-470c-b61b-a8987f3919b5, kind: Service { id: 8e46afa6-9da3-407c-92bf-128b95063f04 }, name: Name(\"nexus-8e46afa6-9da3-407c-92bf-128b95063f04\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ab5e3564-209c-48fa-b8f5-aa370048ac8d, kind: Service { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67 }, name: Name(\"nexus-7b52f14b-12cb-45e4-b4e3-5c517e3cba67\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 39])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: da35a429-5291-41f8-8c2f-7377bb15ed38, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5346055-3249-4357-951e-e5bf8d4dc3cf, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: be250b1e-18db-457d-9646-485f60896fb8, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f545bea-eb7a-44c2-a399-0237abf2a102, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e35814c6-9e18-4226-86a8-2ab44a0293b2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 82bbc823-811c-4ea4-adbb-453cfe570acf, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42568398-8d6b-43e8-b33e-d40f8662e422, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External } } } }, OmicronZoneConfig { id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41863173-79c0-4807-9ede-f5bf31ee3536, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68e6a369-307f-4afa-a667-cfb114baa982, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.945278605Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.94563169Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.9456466Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.967387198Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.967432498Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.986138837Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:26.986184418Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.004806045Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.004866386Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.024776044Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.024824345Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.043577684Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.043636735Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.061844706Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.061887576Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.083007094Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.083056594Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.101735863Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.101786673Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.122525005Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.122578325Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.142471003Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.142521384Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.16230526Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.162358191Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.18111952Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.181166341Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.199736828Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.199794169Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.218313224Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.218367615Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.237258337Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.237302848Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.255953275Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.256007456Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.274918438Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.274962839Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.29383027Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.293879241Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.315624508Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.315699019Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.33952823Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.339587011Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.339837395Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.340001497Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:27.44289914Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:27.442961171Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:27.442968252Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:27.442973861Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:27.442979652Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:27.442985452Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:27.442990792Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:27.442995982Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:27.443005872Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:27.443013162Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:27.443029842Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:27.443033893Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:27.443038182Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:27.443043773Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:27.443047403Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:27.443050923Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:27.443054693Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:27.443059833Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:27.443065063Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.443142784Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"df00d368-78ee-4e7b-98c5-76235ed849c7","remote_addr":"[fd00:1122:3344:101::1]:35174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\", TimeNotSynchronized), (\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\", TimeNotSynchronized), (\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\", TimeNotSynchronized), (\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\", TimeNotSynchronized), (\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\", TimeNotSynchronized), (\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\", TimeNotSynchronized), (\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\", TimeNotSynchronized), (\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\", TimeNotSynchronized), (\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\", TimeNotSynchronized), (\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\", TimeNotSynchronized), (\"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\", TimeNotSynchronized), (\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\", TimeNotSynchronized), (\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\", TimeNotSynchronized), (\"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf\", TimeNotSynchronized), (\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\", TimeNotSynchronized), (\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\", TimeNotSynchronized), (\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\", TimeNotSynchronized), (\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\", TimeNotSynchronized)]","latency_us":497704,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:27.443355828Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"364.937767ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"df00d368-78ee-4e7b-98c5-76235ed849c7\", \"content-length\": \"124\", \"date\": \"Sat, 06 Jul 2024 00:04:26 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"df00d368-78ee-4e7b-98c5-76235ed849c7\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: cdec4b62-7244-4b80-8777-6d905a5202cd, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 92bc3f3a-c64a-4323-b745-aba7d1b54a8d, kind: Service { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a }, name: Name(\"ntp-af96c243-6c6a-4cb4-8eab-8f8a25fd030a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 160])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f85a133-0257-47cf-8244-149e630b6fde, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: e38bc018-21d0-4daa-8b2c-d199004af618, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 69cbd71e-481b-406f-9eb3-20fa7a960393, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 544a0264-7c3a-4ada-aa6e-553800c85da2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 4d9df03f-9907-4449-8d22-3a38f32b899c, kind: Service { id: 544a0264-7c3a-4ada-aa6e-553800c85da2 }, name: Name(\"external-dns-544a0264-7c3a-4ada-aa6e-553800c85da2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 114])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: f0ddfaba-8652-4628-848a-f993e079fb94, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 0ab9fb64-2e84-44e1-91ff-d472c798d3d1, kind: Service { id: f0ddfaba-8652-4628-848a-f993e079fb94 }, name: Name(\"external-dns-f0ddfaba-8652-4628-848a-f993e079fb94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 96bac47d-0e2f-4191-820f-fdc11f7f9342, kind: Service { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f }, name: Name(\"nexus-9ff5ab27-a3ab-4401-8a23-c0799de85f2f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e46afa6-9da3-407c-92bf-128b95063f04, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4ad4ab3a-b527-470c-b61b-a8987f3919b5, kind: Service { id: 8e46afa6-9da3-407c-92bf-128b95063f04 }, name: Name(\"nexus-8e46afa6-9da3-407c-92bf-128b95063f04\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ab5e3564-209c-48fa-b8f5-aa370048ac8d, kind: Service { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67 }, name: Name(\"nexus-7b52f14b-12cb-45e4-b4e3-5c517e3cba67\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 39])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: da35a429-5291-41f8-8c2f-7377bb15ed38, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5346055-3249-4357-951e-e5bf8d4dc3cf, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: be250b1e-18db-457d-9646-485f60896fb8, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f545bea-eb7a-44c2-a399-0237abf2a102, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e35814c6-9e18-4226-86a8-2ab44a0293b2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 82bbc823-811c-4ea4-adbb-453cfe570acf, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42568398-8d6b-43e8-b33e-d40f8662e422, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External } } } }, OmicronZoneConfig { id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41863173-79c0-4807-9ede-f5bf31ee3536, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68e6a369-307f-4afa-a667-cfb114baa982, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.808717812Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.809063497Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.809071167Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.832558012Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.832611123Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.85242764Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.8524762Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.873340433Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.873397554Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.89503807Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.895083101Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.917569469Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.91761261Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.936661494Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.936715765Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.955168829Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.95521202Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.976033332Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.976082543Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.997721219Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:27.99777294Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.020009725Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.020059406Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.041884004Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.041945945Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.064497255Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.064543696Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.0848265Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.084884471Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.104485764Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.104532154Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.123488697Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.123530608Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.146644947Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.146692538Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.167567841Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.167613612Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.187288666Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.187337557Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.207522109Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.20756564Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.227224374Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.227277624Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.227475467Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.227584759Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:28.327606036Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:28.327669698Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:28.327678278Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:28.327683708Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:28.327691378Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:28.327697268Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:28.327705368Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:28.327712798Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:28.327719138Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:28.327724928Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:28.327732039Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:28.327738228Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:28.327745739Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:28.327752059Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:28.327768949Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:28.327773289Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:28.327780309Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:28.327785799Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:28.327791089Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.327917051Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"32016dd4-40cb-4dc4-b8de-844ad1443fc8","remote_addr":"[fd00:1122:3344:101::1]:35174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\", TimeNotSynchronized), (\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\", TimeNotSynchronized), (\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\", TimeNotSynchronized), (\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\", TimeNotSynchronized), (\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\", TimeNotSynchronized), (\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\", TimeNotSynchronized), (\"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf\", TimeNotSynchronized), (\"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67\", TimeNotSynchronized), (\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\", TimeNotSynchronized), (\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\", TimeNotSynchronized), (\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\", TimeNotSynchronized), (\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\", TimeNotSynchronized), (\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\", TimeNotSynchronized), (\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\", TimeNotSynchronized), (\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\", TimeNotSynchronized), (\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\", TimeNotSynchronized), (\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\", TimeNotSynchronized), (\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\", TimeNotSynchronized)]","latency_us":519060,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:28.328152285Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"184.515442ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"32016dd4-40cb-4dc4-b8de-844ad1443fc8\", \"content-length\": \"124\", \"date\": \"Sat, 06 Jul 2024 00:04:28 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"32016dd4-40cb-4dc4-b8de-844ad1443fc8\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: cdec4b62-7244-4b80-8777-6d905a5202cd, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 92bc3f3a-c64a-4323-b745-aba7d1b54a8d, kind: Service { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a }, name: Name(\"ntp-af96c243-6c6a-4cb4-8eab-8f8a25fd030a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 160])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f85a133-0257-47cf-8244-149e630b6fde, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: e38bc018-21d0-4daa-8b2c-d199004af618, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 69cbd71e-481b-406f-9eb3-20fa7a960393, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 544a0264-7c3a-4ada-aa6e-553800c85da2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 4d9df03f-9907-4449-8d22-3a38f32b899c, kind: Service { id: 544a0264-7c3a-4ada-aa6e-553800c85da2 }, name: Name(\"external-dns-544a0264-7c3a-4ada-aa6e-553800c85da2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 114])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: f0ddfaba-8652-4628-848a-f993e079fb94, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 0ab9fb64-2e84-44e1-91ff-d472c798d3d1, kind: Service { id: f0ddfaba-8652-4628-848a-f993e079fb94 }, name: Name(\"external-dns-f0ddfaba-8652-4628-848a-f993e079fb94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 96bac47d-0e2f-4191-820f-fdc11f7f9342, kind: Service { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f }, name: Name(\"nexus-9ff5ab27-a3ab-4401-8a23-c0799de85f2f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e46afa6-9da3-407c-92bf-128b95063f04, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4ad4ab3a-b527-470c-b61b-a8987f3919b5, kind: Service { id: 8e46afa6-9da3-407c-92bf-128b95063f04 }, name: Name(\"nexus-8e46afa6-9da3-407c-92bf-128b95063f04\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ab5e3564-209c-48fa-b8f5-aa370048ac8d, kind: Service { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67 }, name: Name(\"nexus-7b52f14b-12cb-45e4-b4e3-5c517e3cba67\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 39])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: da35a429-5291-41f8-8c2f-7377bb15ed38, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5346055-3249-4357-951e-e5bf8d4dc3cf, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: be250b1e-18db-457d-9646-485f60896fb8, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f545bea-eb7a-44c2-a399-0237abf2a102, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e35814c6-9e18-4226-86a8-2ab44a0293b2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 82bbc823-811c-4ea4-adbb-453cfe570acf, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42568398-8d6b-43e8-b33e-d40f8662e422, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External } } } }, OmicronZoneConfig { id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41863173-79c0-4807-9ede-f5bf31ee3536, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68e6a369-307f-4afa-a667-cfb114baa982, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.51444491Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.514799015Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.514811466Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.538785429Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.538837879Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.560922412Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.560973453Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.582014009Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.58207925Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.601980498Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.602031028Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.621943336Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.621992237Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.643799095Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.643848326Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.664446825Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.664495096Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.685043544Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.685093415Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.706363924Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.706408295Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.728977466Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.729025796Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.751363393Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.751410074Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.770301205Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.770380176Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.791079687Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.791126058Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.81191395Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.81195942Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.832327606Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.832375077Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.8526565Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.852704481Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.872774351Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.872830003Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.892762211Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.892808131Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.913584562Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.913629823Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.933292997Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.933356268Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.933565921Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:28.933702754Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:29.031890582Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:29.031943282Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:29.031949132Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:29.031974693Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:29.031980823Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:29.031986733Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:29.031992693Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:29.031998533Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:29.032004193Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:29.032010993Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:29.032016613Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:29.032022153Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:29.032028014Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:29.032033804Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:29.032039234Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:29.032061914Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:29.032073994Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:29.032079174Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:29.032088775Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:29.032271268Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"a0f22d37-7e2b-4126-8b43-ec5d6f810103","remote_addr":"[fd00:1122:3344:101::1]:35174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\", TimeNotSynchronized), (\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\", TimeNotSynchronized), (\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\", TimeNotSynchronized), (\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\", TimeNotSynchronized), (\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\", TimeNotSynchronized), (\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\", TimeNotSynchronized), (\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\", TimeNotSynchronized), (\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\", TimeNotSynchronized), (\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\", TimeNotSynchronized), (\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\", TimeNotSynchronized), (\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\", TimeNotSynchronized), (\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\", TimeNotSynchronized), (\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\", TimeNotSynchronized), (\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\", TimeNotSynchronized), (\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\", TimeNotSynchronized), (\"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67\", TimeNotSynchronized), (\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\", TimeNotSynchronized), (\"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf\", TimeNotSynchronized)]","latency_us":517675,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:29.032532882Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"368.640656ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a0f22d37-7e2b-4126-8b43-ec5d6f810103\", \"content-length\": \"124\", \"date\": \"Sat, 06 Jul 2024 00:04:29 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a0f22d37-7e2b-4126-8b43-ec5d6f810103\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: cdec4b62-7244-4b80-8777-6d905a5202cd, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 92bc3f3a-c64a-4323-b745-aba7d1b54a8d, kind: Service { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a }, name: Name(\"ntp-af96c243-6c6a-4cb4-8eab-8f8a25fd030a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 160])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f85a133-0257-47cf-8244-149e630b6fde, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: e38bc018-21d0-4daa-8b2c-d199004af618, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 69cbd71e-481b-406f-9eb3-20fa7a960393, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 544a0264-7c3a-4ada-aa6e-553800c85da2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 4d9df03f-9907-4449-8d22-3a38f32b899c, kind: Service { id: 544a0264-7c3a-4ada-aa6e-553800c85da2 }, name: Name(\"external-dns-544a0264-7c3a-4ada-aa6e-553800c85da2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 114])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: f0ddfaba-8652-4628-848a-f993e079fb94, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 0ab9fb64-2e84-44e1-91ff-d472c798d3d1, kind: Service { id: f0ddfaba-8652-4628-848a-f993e079fb94 }, name: Name(\"external-dns-f0ddfaba-8652-4628-848a-f993e079fb94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 96bac47d-0e2f-4191-820f-fdc11f7f9342, kind: Service { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f }, name: Name(\"nexus-9ff5ab27-a3ab-4401-8a23-c0799de85f2f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e46afa6-9da3-407c-92bf-128b95063f04, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4ad4ab3a-b527-470c-b61b-a8987f3919b5, kind: Service { id: 8e46afa6-9da3-407c-92bf-128b95063f04 }, name: Name(\"nexus-8e46afa6-9da3-407c-92bf-128b95063f04\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ab5e3564-209c-48fa-b8f5-aa370048ac8d, kind: Service { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67 }, name: Name(\"nexus-7b52f14b-12cb-45e4-b4e3-5c517e3cba67\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 39])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: da35a429-5291-41f8-8c2f-7377bb15ed38, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5346055-3249-4357-951e-e5bf8d4dc3cf, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: be250b1e-18db-457d-9646-485f60896fb8, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f545bea-eb7a-44c2-a399-0237abf2a102, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e35814c6-9e18-4226-86a8-2ab44a0293b2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 82bbc823-811c-4ea4-adbb-453cfe570acf, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42568398-8d6b-43e8-b33e-d40f8662e422, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External } } } }, OmicronZoneConfig { id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41863173-79c0-4807-9ede-f5bf31ee3536, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68e6a369-307f-4afa-a667-cfb114baa982, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:29.402001141Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:29.402359417Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:29.402372157Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:29.42510737Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:29.425161021Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:29.446656474Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:29.446700305Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:29.466344549Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:29.46640125Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:29.485508865Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:29.485571486Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:29.504712352Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:29.504759262Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:29.525060156Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:29.525110067Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:29.544914743Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:29.544965414Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:29.565827897Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:29.565873568Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:29.588820215Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:29.588871135Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:29.610776835Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:29.610820936Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:29.631479756Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:29.631526836Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:29.652829526Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:29.652875147Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:29.673793381Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:29.673851952Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:29.695066501Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:29.695108382Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:29.715797422Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:29.715842303Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:29.737741872Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:29.737792083Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:29.75952768Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:29.759575811Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:29.779769773Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:29.779823734Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:29.800417033Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:29.800461814Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:29.822094109Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:29.82215704Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:29.822393404Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:29.822539836Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:29.921676649Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:29.92175077Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:29.92175964Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:29.92176545Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:29.92177295Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:29.92177925Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:29.921785191Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:29.921808881Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:29.921815171Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:29.921820871Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:29.921826291Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:29.921831981Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:29.921837701Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:29.921844781Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:29.921850852Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:29.921858012Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:29.921863542Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:29.921868742Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:29.921875602Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:29.921997714Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"fde402e2-ee81-4fa1-8df2-29ad96c1b7cc","remote_addr":"[fd00:1122:3344:101::1]:35174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\", TimeNotSynchronized), (\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\", TimeNotSynchronized), (\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\", TimeNotSynchronized), (\"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67\", TimeNotSynchronized), (\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\", TimeNotSynchronized), (\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\", TimeNotSynchronized), (\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\", TimeNotSynchronized), (\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\", TimeNotSynchronized), (\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\", TimeNotSynchronized), (\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\", TimeNotSynchronized), (\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\", TimeNotSynchronized), (\"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf\", TimeNotSynchronized), (\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\", TimeNotSynchronized), (\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\", TimeNotSynchronized), (\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\", TimeNotSynchronized), (\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\", TimeNotSynchronized), (\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\", TimeNotSynchronized), (\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\", TimeNotSynchronized)]","latency_us":519852,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:29.922195117Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"590.813523ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fde402e2-ee81-4fa1-8df2-29ad96c1b7cc\", \"content-length\": \"124\", \"date\": \"Sat, 06 Jul 2024 00:04:29 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"fde402e2-ee81-4fa1-8df2-29ad96c1b7cc\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: cdec4b62-7244-4b80-8777-6d905a5202cd, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 92bc3f3a-c64a-4323-b745-aba7d1b54a8d, kind: Service { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a }, name: Name(\"ntp-af96c243-6c6a-4cb4-8eab-8f8a25fd030a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 160])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f85a133-0257-47cf-8244-149e630b6fde, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: e38bc018-21d0-4daa-8b2c-d199004af618, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 69cbd71e-481b-406f-9eb3-20fa7a960393, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 544a0264-7c3a-4ada-aa6e-553800c85da2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 4d9df03f-9907-4449-8d22-3a38f32b899c, kind: Service { id: 544a0264-7c3a-4ada-aa6e-553800c85da2 }, name: Name(\"external-dns-544a0264-7c3a-4ada-aa6e-553800c85da2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 114])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: f0ddfaba-8652-4628-848a-f993e079fb94, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 0ab9fb64-2e84-44e1-91ff-d472c798d3d1, kind: Service { id: f0ddfaba-8652-4628-848a-f993e079fb94 }, name: Name(\"external-dns-f0ddfaba-8652-4628-848a-f993e079fb94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 96bac47d-0e2f-4191-820f-fdc11f7f9342, kind: Service { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f }, name: Name(\"nexus-9ff5ab27-a3ab-4401-8a23-c0799de85f2f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e46afa6-9da3-407c-92bf-128b95063f04, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4ad4ab3a-b527-470c-b61b-a8987f3919b5, kind: Service { id: 8e46afa6-9da3-407c-92bf-128b95063f04 }, name: Name(\"nexus-8e46afa6-9da3-407c-92bf-128b95063f04\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ab5e3564-209c-48fa-b8f5-aa370048ac8d, kind: Service { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67 }, name: Name(\"nexus-7b52f14b-12cb-45e4-b4e3-5c517e3cba67\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 39])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: da35a429-5291-41f8-8c2f-7377bb15ed38, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5346055-3249-4357-951e-e5bf8d4dc3cf, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: be250b1e-18db-457d-9646-485f60896fb8, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f545bea-eb7a-44c2-a399-0237abf2a102, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e35814c6-9e18-4226-86a8-2ab44a0293b2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 82bbc823-811c-4ea4-adbb-453cfe570acf, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42568398-8d6b-43e8-b33e-d40f8662e422, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External } } } }, OmicronZoneConfig { id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41863173-79c0-4807-9ede-f5bf31ee3536, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68e6a369-307f-4afa-a667-cfb114baa982, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:30.513814734Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:30.514159459Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:30.514167879Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:30.535162455Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:30.535224056Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:30.557995279Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:30.55805395Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:30.576051488Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:30.576109249Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:30.599215408Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:30.599264558Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:30.620538008Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:30.620579139Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:30.642548489Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:30.64259095Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:30.662183913Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:30.662232134Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:30.68266166Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:30.682702741Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:30.704206114Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:30.704250105Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:30.725573935Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:30.725629256Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:30.746900936Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:30.746945236Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:30.768228826Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:30.768280257Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:30.791177013Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:30.791228974Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:30.814514226Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:30.814559496Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:30.83482608Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:30.834884091Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:30.857105826Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:30.857154996Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:30.880036612Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:30.880081882Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:30.901497544Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:30.901546515Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:30.925958605Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:30.926004996Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:30.94758792Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:30.947654171Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:30.947857135Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:30.948007087Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:31.047464535Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:31.047524926Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:31.047533226Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:31.047539386Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:31.047544956Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:31.047550607Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:31.047556127Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:31.047561377Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:31.047568267Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:31.047573757Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:31.047580727Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:31.047587807Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:31.047612777Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:31.047620268Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:31.047625918Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:31.047632668Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:31.047638748Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:31.047646008Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:31.047650188Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:31.047741639Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"7febb39c-73b7-4b6b-be30-d1c3354e3ff4","remote_addr":"[fd00:1122:3344:101::1]:35174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\", TimeNotSynchronized), (\"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\", TimeNotSynchronized), (\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\", TimeNotSynchronized), (\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\", TimeNotSynchronized), (\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\", TimeNotSynchronized), (\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\", TimeNotSynchronized), (\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\", TimeNotSynchronized), (\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\", TimeNotSynchronized), (\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\", TimeNotSynchronized), (\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\", TimeNotSynchronized), (\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\", TimeNotSynchronized), (\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\", TimeNotSynchronized), (\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\", TimeNotSynchronized), (\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\", TimeNotSynchronized), (\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\", TimeNotSynchronized), (\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\", TimeNotSynchronized), (\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\", TimeNotSynchronized), (\"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67\", TimeNotSynchronized)]","latency_us":533778,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:31.047975693Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"513.163317ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7febb39c-73b7-4b6b-be30-d1c3354e3ff4\", \"content-length\": \"124\", \"date\": \"Sat, 06 Jul 2024 00:04:30 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7febb39c-73b7-4b6b-be30-d1c3354e3ff4\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: cdec4b62-7244-4b80-8777-6d905a5202cd, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 92bc3f3a-c64a-4323-b745-aba7d1b54a8d, kind: Service { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a }, name: Name(\"ntp-af96c243-6c6a-4cb4-8eab-8f8a25fd030a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 160])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f85a133-0257-47cf-8244-149e630b6fde, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: e38bc018-21d0-4daa-8b2c-d199004af618, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 69cbd71e-481b-406f-9eb3-20fa7a960393, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 544a0264-7c3a-4ada-aa6e-553800c85da2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 4d9df03f-9907-4449-8d22-3a38f32b899c, kind: Service { id: 544a0264-7c3a-4ada-aa6e-553800c85da2 }, name: Name(\"external-dns-544a0264-7c3a-4ada-aa6e-553800c85da2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 114])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: f0ddfaba-8652-4628-848a-f993e079fb94, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 0ab9fb64-2e84-44e1-91ff-d472c798d3d1, kind: Service { id: f0ddfaba-8652-4628-848a-f993e079fb94 }, name: Name(\"external-dns-f0ddfaba-8652-4628-848a-f993e079fb94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 96bac47d-0e2f-4191-820f-fdc11f7f9342, kind: Service { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f }, name: Name(\"nexus-9ff5ab27-a3ab-4401-8a23-c0799de85f2f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e46afa6-9da3-407c-92bf-128b95063f04, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4ad4ab3a-b527-470c-b61b-a8987f3919b5, kind: Service { id: 8e46afa6-9da3-407c-92bf-128b95063f04 }, name: Name(\"nexus-8e46afa6-9da3-407c-92bf-128b95063f04\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ab5e3564-209c-48fa-b8f5-aa370048ac8d, kind: Service { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67 }, name: Name(\"nexus-7b52f14b-12cb-45e4-b4e3-5c517e3cba67\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 39])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: da35a429-5291-41f8-8c2f-7377bb15ed38, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5346055-3249-4357-951e-e5bf8d4dc3cf, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: be250b1e-18db-457d-9646-485f60896fb8, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f545bea-eb7a-44c2-a399-0237abf2a102, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e35814c6-9e18-4226-86a8-2ab44a0293b2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 82bbc823-811c-4ea4-adbb-453cfe570acf, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42568398-8d6b-43e8-b33e-d40f8662e422, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External } } } }, OmicronZoneConfig { id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41863173-79c0-4807-9ede-f5bf31ee3536, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68e6a369-307f-4afa-a667-cfb114baa982, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:31.562364937Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:31.562728163Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:31.562735453Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:31.586617554Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:31.586680985Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:31.608058437Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:31.608111847Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:31.631006793Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:31.631060444Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:31.651104204Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:31.651156725Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:31.670109077Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:31.670156308Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:31.689750651Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:31.689798452Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:31.708788985Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:31.708855676Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:31.728670132Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:31.728716003Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:31.749842541Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:31.749884051Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:31.770982438Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:31.771032359Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:31.791813331Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:31.791863132Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:31.813242483Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:31.813289944Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:31.834140547Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:31.834196088Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:31.854457211Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:31.854507812Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:31.875953664Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:31.875995845Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:31.897994496Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:31.898049857Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:31.920652728Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:31.920693028Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:31.94151255Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:31.941570482Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:31.962708629Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:31.96275445Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:31.984732031Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:31.984796592Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:31.985022465Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:31.985181658Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:32.083598569Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:32.083680221Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:32.083688981Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:32.083694631Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:32.083700421Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:32.083705851Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:32.083712661Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:32.083718651Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:32.083724801Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:32.083731652Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:32.083735841Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:32.083741722Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:32.083747142Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:32.083752642Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:32.083758132Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:32.083763752Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:32.083785202Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:32.083792122Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:32.083797353Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:32.083929475Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"739fd38d-7d6a-4f74-bf53-c305acbf1953","remote_addr":"[fd00:1122:3344:101::1]:35174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\", TimeNotSynchronized), (\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\", TimeNotSynchronized), (\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\", TimeNotSynchronized), (\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\", TimeNotSynchronized), (\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\", TimeNotSynchronized), (\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\", TimeNotSynchronized), (\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\", TimeNotSynchronized), (\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\", TimeNotSynchronized), (\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\", TimeNotSynchronized), (\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\", TimeNotSynchronized), (\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\", TimeNotSynchronized), (\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\", TimeNotSynchronized), (\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\", TimeNotSynchronized), (\"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\", TimeNotSynchronized), (\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\", TimeNotSynchronized), (\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\", TimeNotSynchronized), (\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\", TimeNotSynchronized), (\"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf\", TimeNotSynchronized)]","latency_us":521412,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:32.084135118Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"760.551531ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"739fd38d-7d6a-4f74-bf53-c305acbf1953\", \"content-length\": \"124\", \"date\": \"Sat, 06 Jul 2024 00:04:31 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"739fd38d-7d6a-4f74-bf53-c305acbf1953\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: cdec4b62-7244-4b80-8777-6d905a5202cd, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 92bc3f3a-c64a-4323-b745-aba7d1b54a8d, kind: Service { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a }, name: Name(\"ntp-af96c243-6c6a-4cb4-8eab-8f8a25fd030a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 160])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f85a133-0257-47cf-8244-149e630b6fde, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: e38bc018-21d0-4daa-8b2c-d199004af618, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 69cbd71e-481b-406f-9eb3-20fa7a960393, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 544a0264-7c3a-4ada-aa6e-553800c85da2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 4d9df03f-9907-4449-8d22-3a38f32b899c, kind: Service { id: 544a0264-7c3a-4ada-aa6e-553800c85da2 }, name: Name(\"external-dns-544a0264-7c3a-4ada-aa6e-553800c85da2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 114])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: f0ddfaba-8652-4628-848a-f993e079fb94, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 0ab9fb64-2e84-44e1-91ff-d472c798d3d1, kind: Service { id: f0ddfaba-8652-4628-848a-f993e079fb94 }, name: Name(\"external-dns-f0ddfaba-8652-4628-848a-f993e079fb94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 96bac47d-0e2f-4191-820f-fdc11f7f9342, kind: Service { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f }, name: Name(\"nexus-9ff5ab27-a3ab-4401-8a23-c0799de85f2f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e46afa6-9da3-407c-92bf-128b95063f04, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4ad4ab3a-b527-470c-b61b-a8987f3919b5, kind: Service { id: 8e46afa6-9da3-407c-92bf-128b95063f04 }, name: Name(\"nexus-8e46afa6-9da3-407c-92bf-128b95063f04\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ab5e3564-209c-48fa-b8f5-aa370048ac8d, kind: Service { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67 }, name: Name(\"nexus-7b52f14b-12cb-45e4-b4e3-5c517e3cba67\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 39])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: da35a429-5291-41f8-8c2f-7377bb15ed38, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5346055-3249-4357-951e-e5bf8d4dc3cf, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: be250b1e-18db-457d-9646-485f60896fb8, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f545bea-eb7a-44c2-a399-0237abf2a102, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e35814c6-9e18-4226-86a8-2ab44a0293b2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 82bbc823-811c-4ea4-adbb-453cfe570acf, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42568398-8d6b-43e8-b33e-d40f8662e422, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External } } } }, OmicronZoneConfig { id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41863173-79c0-4807-9ede-f5bf31ee3536, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68e6a369-307f-4afa-a667-cfb114baa982, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:32.84580468Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:32.846152276Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:32.846160056Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:32.869035121Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:32.869089652Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:32.892956343Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:32.893007444Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:32.912648137Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:32.912693757Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:32.932959071Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:32.933003422Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:32.952392481Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:32.952433652Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:32.972176907Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:32.972225458Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:32.992328319Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:32.992366909Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:33.011311612Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:33.011373193Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:33.033472056Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:33.033519457Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:33.054109546Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:33.054156546Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:33.074966759Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:33.075011139Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:33.096778197Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:33.096830988Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:33.116708085Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:33.116758386Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:33.137985945Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:33.138033466Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:33.158213438Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:33.158260229Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:33.178667594Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:33.178709355Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:33.202379523Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:33.202430504Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:33.223753584Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:33.223806845Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:33.245459741Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:33.245514122Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:33.267220728Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:33.267280909Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:33.267499253Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:33.267642505Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:33.362580271Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:33.362639392Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:33.362647922Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:33.362652492Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:33.362659322Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:33.362676433Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:33.362683873Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:33.362689023Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:33.362695893Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:33.362701433Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:33.362706773Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:33.362713663Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:33.362718893Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:33.362725663Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:33.362731174Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:33.362736794Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:33.362743604Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:33.362749044Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:33.362754764Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:33.362835855Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"a50f3873-d666-4178-836f-320b467cf295","remote_addr":"[fd00:1122:3344:101::1]:35174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\", TimeNotSynchronized), (\"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\", TimeNotSynchronized), (\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\", TimeNotSynchronized), (\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\", TimeNotSynchronized), (\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\", TimeNotSynchronized), (\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\", TimeNotSynchronized), (\"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf\", TimeNotSynchronized), (\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\", TimeNotSynchronized), (\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\", TimeNotSynchronized), (\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\", TimeNotSynchronized), (\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\", TimeNotSynchronized), (\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\", TimeNotSynchronized), (\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\", TimeNotSynchronized), (\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\", TimeNotSynchronized), (\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\", TimeNotSynchronized), (\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\", TimeNotSynchronized), (\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\", TimeNotSynchronized), (\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\", TimeNotSynchronized)]","latency_us":516891,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:33.363059449Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"470.930495ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a50f3873-d666-4178-836f-320b467cf295\", \"content-length\": \"124\", \"date\": \"Sat, 06 Jul 2024 00:04:32 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a50f3873-d666-4178-836f-320b467cf295\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: cdec4b62-7244-4b80-8777-6d905a5202cd, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 92bc3f3a-c64a-4323-b745-aba7d1b54a8d, kind: Service { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a }, name: Name(\"ntp-af96c243-6c6a-4cb4-8eab-8f8a25fd030a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 160])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f85a133-0257-47cf-8244-149e630b6fde, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: e38bc018-21d0-4daa-8b2c-d199004af618, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 69cbd71e-481b-406f-9eb3-20fa7a960393, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 544a0264-7c3a-4ada-aa6e-553800c85da2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 4d9df03f-9907-4449-8d22-3a38f32b899c, kind: Service { id: 544a0264-7c3a-4ada-aa6e-553800c85da2 }, name: Name(\"external-dns-544a0264-7c3a-4ada-aa6e-553800c85da2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 114])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: f0ddfaba-8652-4628-848a-f993e079fb94, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 0ab9fb64-2e84-44e1-91ff-d472c798d3d1, kind: Service { id: f0ddfaba-8652-4628-848a-f993e079fb94 }, name: Name(\"external-dns-f0ddfaba-8652-4628-848a-f993e079fb94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 96bac47d-0e2f-4191-820f-fdc11f7f9342, kind: Service { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f }, name: Name(\"nexus-9ff5ab27-a3ab-4401-8a23-c0799de85f2f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e46afa6-9da3-407c-92bf-128b95063f04, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4ad4ab3a-b527-470c-b61b-a8987f3919b5, kind: Service { id: 8e46afa6-9da3-407c-92bf-128b95063f04 }, name: Name(\"nexus-8e46afa6-9da3-407c-92bf-128b95063f04\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ab5e3564-209c-48fa-b8f5-aa370048ac8d, kind: Service { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67 }, name: Name(\"nexus-7b52f14b-12cb-45e4-b4e3-5c517e3cba67\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 39])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: da35a429-5291-41f8-8c2f-7377bb15ed38, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5346055-3249-4357-951e-e5bf8d4dc3cf, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: be250b1e-18db-457d-9646-485f60896fb8, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f545bea-eb7a-44c2-a399-0237abf2a102, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e35814c6-9e18-4226-86a8-2ab44a0293b2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 82bbc823-811c-4ea4-adbb-453cfe570acf, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42568398-8d6b-43e8-b33e-d40f8662e422, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External } } } }, OmicronZoneConfig { id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41863173-79c0-4807-9ede-f5bf31ee3536, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68e6a369-307f-4afa-a667-cfb114baa982, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:33.835427831Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:33.835772767Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:33.835790607Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:33.858222055Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:33.858275856Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:33.880175606Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:33.880222817Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:33.901928493Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:33.901979464Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:33.922111086Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:33.922157566Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:33.94309299Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:33.943156191Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:33.965891525Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:33.965934935Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:33.987701193Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:33.987748134Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:34.007644051Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:34.007688752Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:34.031243208Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:34.031292519Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:34.05391137Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:34.053956891Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:34.07583966Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:34.075884911Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:34.096251686Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:34.096299397Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:34.117289632Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:34.117341463Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:34.137772039Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:34.13782438Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:34.161944885Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:34.161990876Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:34.183471689Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:34.18351729Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:34.20920469Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:34.209256801Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:34.23366004Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:34.233701281Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:34.256230721Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:34.256283751Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:34.279612894Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:34.279664025Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:34.279872048Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:34.27999426Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:34.37828874Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:34.378353021Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:34.378359981Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:34.378367251Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:34.378372941Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:34.378378501Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:34.378384461Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:34.378391721Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:34.378397472Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:34.378414442Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:34.378422512Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:34.378427892Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:34.378433152Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:34.378439972Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:34.378445362Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:34.378450682Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:34.378456172Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:34.378461462Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:34.378466493Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:34.378566634Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"514840db-e8c8-4b94-ac0f-fd33e1e3beed","remote_addr":"[fd00:1122:3344:101::1]:35174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\", TimeNotSynchronized), (\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\", TimeNotSynchronized), (\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\", TimeNotSynchronized), (\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\", TimeNotSynchronized), (\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\", TimeNotSynchronized), (\"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf\", TimeNotSynchronized), (\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\", TimeNotSynchronized), (\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\", TimeNotSynchronized), (\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\", TimeNotSynchronized), (\"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67\", TimeNotSynchronized), (\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\", TimeNotSynchronized), (\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\", TimeNotSynchronized), (\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\", TimeNotSynchronized), (\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\", TimeNotSynchronized), (\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\", TimeNotSynchronized), (\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\", TimeNotSynchronized), (\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\", TimeNotSynchronized), (\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\", TimeNotSynchronized)]","latency_us":543000,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:34.378784778Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"1.260549504s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"514840db-e8c8-4b94-ac0f-fd33e1e3beed\", \"content-length\": \"124\", \"date\": \"Sat, 06 Jul 2024 00:04:34 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"514840db-e8c8-4b94-ac0f-fd33e1e3beed\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: cdec4b62-7244-4b80-8777-6d905a5202cd, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 92bc3f3a-c64a-4323-b745-aba7d1b54a8d, kind: Service { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a }, name: Name(\"ntp-af96c243-6c6a-4cb4-8eab-8f8a25fd030a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 160])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f85a133-0257-47cf-8244-149e630b6fde, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: e38bc018-21d0-4daa-8b2c-d199004af618, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 69cbd71e-481b-406f-9eb3-20fa7a960393, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 544a0264-7c3a-4ada-aa6e-553800c85da2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 4d9df03f-9907-4449-8d22-3a38f32b899c, kind: Service { id: 544a0264-7c3a-4ada-aa6e-553800c85da2 }, name: Name(\"external-dns-544a0264-7c3a-4ada-aa6e-553800c85da2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 114])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: f0ddfaba-8652-4628-848a-f993e079fb94, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 0ab9fb64-2e84-44e1-91ff-d472c798d3d1, kind: Service { id: f0ddfaba-8652-4628-848a-f993e079fb94 }, name: Name(\"external-dns-f0ddfaba-8652-4628-848a-f993e079fb94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 96bac47d-0e2f-4191-820f-fdc11f7f9342, kind: Service { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f }, name: Name(\"nexus-9ff5ab27-a3ab-4401-8a23-c0799de85f2f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e46afa6-9da3-407c-92bf-128b95063f04, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4ad4ab3a-b527-470c-b61b-a8987f3919b5, kind: Service { id: 8e46afa6-9da3-407c-92bf-128b95063f04 }, name: Name(\"nexus-8e46afa6-9da3-407c-92bf-128b95063f04\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ab5e3564-209c-48fa-b8f5-aa370048ac8d, kind: Service { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67 }, name: Name(\"nexus-7b52f14b-12cb-45e4-b4e3-5c517e3cba67\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 39])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: da35a429-5291-41f8-8c2f-7377bb15ed38, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5346055-3249-4357-951e-e5bf8d4dc3cf, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: be250b1e-18db-457d-9646-485f60896fb8, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f545bea-eb7a-44c2-a399-0237abf2a102, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e35814c6-9e18-4226-86a8-2ab44a0293b2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 82bbc823-811c-4ea4-adbb-453cfe570acf, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42568398-8d6b-43e8-b33e-d40f8662e422, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External } } } }, OmicronZoneConfig { id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41863173-79c0-4807-9ede-f5bf31ee3536, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68e6a369-307f-4afa-a667-cfb114baa982, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:35.640585385Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:35.640937941Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:35.640947741Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:35.667738689Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:35.66779783Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:35.688764835Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:35.688815175Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:35.707757268Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:35.707814459Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:35.728784794Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:35.728836744Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:35.749378712Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:35.749427093Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:35.770143594Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:35.770207565Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:35.790870435Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:35.790924876Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:35.812086424Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:35.812132614Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:35.833417344Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:35.833464385Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:35.85572239Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:35.855778131Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:35.878961341Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:35.879024302Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:35.90075887Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:35.90080512Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:35.922402374Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:35.922451895Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:35.946685962Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:35.946736003Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:35.96849246Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:35.968540601Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:35.990341759Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:35.99039604Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:36.012382922Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:36.012429564Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:36.035150273Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:36.03519609Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:36.058202674Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:36.058250196Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:36.081493356Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:36.0815506Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:36.081765835Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:36.081905319Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:36.181031709Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:36.181093032Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:36.18110176Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:36.181109738Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:36.18111618Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:36.181122938Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:36.181129868Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:36.181134893Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:36.18114138Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:36.181146349Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:36.181151139Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:36.181156211Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:36.181159924Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:36.181165005Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:36.181177558Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:36.181181225Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:36.181186015Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:36.181190731Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:36.181195775Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:36.181260144Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"afdb1636-5097-41aa-b46f-2f300e659a32","remote_addr":"[fd00:1122:3344:101::1]:35174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\", TimeNotSynchronized), (\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\", TimeNotSynchronized), (\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\", TimeNotSynchronized), (\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\", TimeNotSynchronized), (\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\", TimeNotSynchronized), (\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\", TimeNotSynchronized), (\"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf\", TimeNotSynchronized), (\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\", TimeNotSynchronized), (\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\", TimeNotSynchronized), (\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\", TimeNotSynchronized), (\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\", TimeNotSynchronized), (\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\", TimeNotSynchronized), (\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\", TimeNotSynchronized), (\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\", TimeNotSynchronized), (\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\", TimeNotSynchronized), (\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\", TimeNotSynchronized), (\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\", TimeNotSynchronized), (\"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67\", TimeNotSynchronized)]","latency_us":540093,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:36.181528011Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"1.272753459s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"afdb1636-5097-41aa-b46f-2f300e659a32\", \"content-length\": \"124\", \"date\": \"Sat, 06 Jul 2024 00:04:35 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"afdb1636-5097-41aa-b46f-2f300e659a32\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: cdec4b62-7244-4b80-8777-6d905a5202cd, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 92bc3f3a-c64a-4323-b745-aba7d1b54a8d, kind: Service { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a }, name: Name(\"ntp-af96c243-6c6a-4cb4-8eab-8f8a25fd030a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 160])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f85a133-0257-47cf-8244-149e630b6fde, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: e38bc018-21d0-4daa-8b2c-d199004af618, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 69cbd71e-481b-406f-9eb3-20fa7a960393, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 544a0264-7c3a-4ada-aa6e-553800c85da2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 4d9df03f-9907-4449-8d22-3a38f32b899c, kind: Service { id: 544a0264-7c3a-4ada-aa6e-553800c85da2 }, name: Name(\"external-dns-544a0264-7c3a-4ada-aa6e-553800c85da2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 114])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: f0ddfaba-8652-4628-848a-f993e079fb94, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 0ab9fb64-2e84-44e1-91ff-d472c798d3d1, kind: Service { id: f0ddfaba-8652-4628-848a-f993e079fb94 }, name: Name(\"external-dns-f0ddfaba-8652-4628-848a-f993e079fb94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 96bac47d-0e2f-4191-820f-fdc11f7f9342, kind: Service { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f }, name: Name(\"nexus-9ff5ab27-a3ab-4401-8a23-c0799de85f2f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e46afa6-9da3-407c-92bf-128b95063f04, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4ad4ab3a-b527-470c-b61b-a8987f3919b5, kind: Service { id: 8e46afa6-9da3-407c-92bf-128b95063f04 }, name: Name(\"nexus-8e46afa6-9da3-407c-92bf-128b95063f04\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ab5e3564-209c-48fa-b8f5-aa370048ac8d, kind: Service { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67 }, name: Name(\"nexus-7b52f14b-12cb-45e4-b4e3-5c517e3cba67\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 39])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: da35a429-5291-41f8-8c2f-7377bb15ed38, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5346055-3249-4357-951e-e5bf8d4dc3cf, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: be250b1e-18db-457d-9646-485f60896fb8, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f545bea-eb7a-44c2-a399-0237abf2a102, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e35814c6-9e18-4226-86a8-2ab44a0293b2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 82bbc823-811c-4ea4-adbb-453cfe570acf, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42568398-8d6b-43e8-b33e-d40f8662e422, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External } } } }, OmicronZoneConfig { id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41863173-79c0-4807-9ede-f5bf31ee3536, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68e6a369-307f-4afa-a667-cfb114baa982, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:37.459054474Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:37.459390198Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:37.459396863Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:37.483680278Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:37.483728512Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:37.504789381Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:37.504835807Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:37.525037243Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:37.525165333Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:37.545601636Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:37.54564614Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:37.565254636Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:37.565298333Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:37.585042448Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:37.585090279Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:37.604806424Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:37.604845078Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:37.627631406Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:37.627675807Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:37.649342582Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:37.64945351Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:37.669599563Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:37.669639567Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:37.691967264Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:37.692005496Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:37.712335689Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:37.71238684Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:37.733332957Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:37.733375671Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:37.753574318Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:37.753611724Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:37.773510757Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:37.773552627Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:37.794480964Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:37.79452512Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:37.814441321Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:37.81448994Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:37.83498273Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:37.835026296Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:37.856159972Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:37.85619991Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:37.877255998Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:37.877306717Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:37.877498036Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:37.877620757Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:37.979983036Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:37.9800415Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:37.980049018Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:37.980071519Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:37.980075382Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:37.980080707Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:37.980085929Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:37.980090973Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:37.980096157Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:37.980101501Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:37.980107913Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:37.980112986Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:37.980119511Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:37.980124705Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:37.980131277Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:37.980136245Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:37.980141477Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:37.980146736Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:37.980156758Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:37.980249732Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"d5bdce4f-1d02-4c4d-b519-c0b847b8de87","remote_addr":"[fd00:1122:3344:101::1]:35174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\", TimeNotSynchronized), (\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\", TimeNotSynchronized), (\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\", TimeNotSynchronized), (\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\", TimeNotSynchronized), (\"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf\", TimeNotSynchronized), (\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\", TimeNotSynchronized), (\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\", TimeNotSynchronized), (\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\", TimeNotSynchronized), (\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\", TimeNotSynchronized), (\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\", TimeNotSynchronized), (\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\", TimeNotSynchronized), (\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\", TimeNotSynchronized), (\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\", TimeNotSynchronized), (\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\", TimeNotSynchronized), (\"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67\", TimeNotSynchronized), (\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\", TimeNotSynchronized), (\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\", TimeNotSynchronized), (\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\", TimeNotSynchronized)]","latency_us":519698,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:37.980536051Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"1.906001152s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d5bdce4f-1d02-4c4d-b519-c0b847b8de87\", \"content-length\": \"124\", \"date\": \"Sat, 06 Jul 2024 00:04:37 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d5bdce4f-1d02-4c4d-b519-c0b847b8de87\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:38.988235933Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:423","component":"producer-server","component":"MetricsManager","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:423","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"1.650751273s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: cdec4b62-7244-4b80-8777-6d905a5202cd, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 92bc3f3a-c64a-4323-b745-aba7d1b54a8d, kind: Service { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a }, name: Name(\"ntp-af96c243-6c6a-4cb4-8eab-8f8a25fd030a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 160])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f85a133-0257-47cf-8244-149e630b6fde, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: e38bc018-21d0-4daa-8b2c-d199004af618, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 69cbd71e-481b-406f-9eb3-20fa7a960393, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 544a0264-7c3a-4ada-aa6e-553800c85da2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 4d9df03f-9907-4449-8d22-3a38f32b899c, kind: Service { id: 544a0264-7c3a-4ada-aa6e-553800c85da2 }, name: Name(\"external-dns-544a0264-7c3a-4ada-aa6e-553800c85da2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 114])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: f0ddfaba-8652-4628-848a-f993e079fb94, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 0ab9fb64-2e84-44e1-91ff-d472c798d3d1, kind: Service { id: f0ddfaba-8652-4628-848a-f993e079fb94 }, name: Name(\"external-dns-f0ddfaba-8652-4628-848a-f993e079fb94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 96bac47d-0e2f-4191-820f-fdc11f7f9342, kind: Service { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f }, name: Name(\"nexus-9ff5ab27-a3ab-4401-8a23-c0799de85f2f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e46afa6-9da3-407c-92bf-128b95063f04, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4ad4ab3a-b527-470c-b61b-a8987f3919b5, kind: Service { id: 8e46afa6-9da3-407c-92bf-128b95063f04 }, name: Name(\"nexus-8e46afa6-9da3-407c-92bf-128b95063f04\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ab5e3564-209c-48fa-b8f5-aa370048ac8d, kind: Service { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67 }, name: Name(\"nexus-7b52f14b-12cb-45e4-b4e3-5c517e3cba67\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 39])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: da35a429-5291-41f8-8c2f-7377bb15ed38, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5346055-3249-4357-951e-e5bf8d4dc3cf, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: be250b1e-18db-457d-9646-485f60896fb8, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f545bea-eb7a-44c2-a399-0237abf2a102, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e35814c6-9e18-4226-86a8-2ab44a0293b2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 82bbc823-811c-4ea4-adbb-453cfe570acf, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42568398-8d6b-43e8-b33e-d40f8662e422, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External } } } }, OmicronZoneConfig { id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41863173-79c0-4807-9ede-f5bf31ee3536, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68e6a369-307f-4afa-a667-cfb114baa982, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:39.892149436Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:39.892436288Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:39.89251573Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:39.915359069Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:39.915407595Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:39.93681262Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:39.936860265Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:39.957216987Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:39.95726691Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:39.977604706Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:39.977649716Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:39.999542479Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:39.999589992Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:40.019951667Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:40.019996415Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:40.039467027Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:40.039512825Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:40.059401497Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:40.059461329Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:40.080561322Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:40.080607916Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:40.099510919Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:40.099556323Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:40.120129884Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:40.120172232Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:40.14062947Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:40.140674114Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:40.161317476Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:40.161386158Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:40.18360764Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:40.183658547Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:40.205608384Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:40.20565195Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:40.226904563Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:40.226950342Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:40.246602789Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:40.246650349Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:40.268067856Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:40.26812195Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:40.288048057Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:40.288092954Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:40.31070724Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:40.310762132Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:40.31097672Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:40.311129047Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:40.409699761Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:40.409752477Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:40.40976039Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:40.409767721Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:40.409773224Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:40.409799241Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:40.409806009Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:40.409811156Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:40.409816013Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:40.409820888Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:40.409827019Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:40.409832335Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:40.409838832Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:40.40990876Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:40.409913851Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:40.409920104Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:40.409925101Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:40.409930416Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:40.409934176Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:40.410010592Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"e480fb71-89e7-4d1c-9ebc-c6988a973d9c","remote_addr":"[fd00:1122:3344:101::1]:35174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\", TimeNotSynchronized), (\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\", TimeNotSynchronized), (\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\", TimeNotSynchronized), (\"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67\", TimeNotSynchronized), (\"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf\", TimeNotSynchronized), (\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\", TimeNotSynchronized), (\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\", TimeNotSynchronized), (\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\", TimeNotSynchronized), (\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\", TimeNotSynchronized), (\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\", TimeNotSynchronized), (\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\", TimeNotSynchronized), (\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\", TimeNotSynchronized), (\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\", TimeNotSynchronized), (\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\", TimeNotSynchronized), (\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\", TimeNotSynchronized), (\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\", TimeNotSynchronized), (\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\", TimeNotSynchronized), (\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\", TimeNotSynchronized)]","latency_us":516301,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:40.410220718Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"2.221888585s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e480fb71-89e7-4d1c-9ebc-c6988a973d9c\", \"content-length\": \"124\", \"date\": \"Sat, 06 Jul 2024 00:04:39 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e480fb71-89e7-4d1c-9ebc-c6988a973d9c\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: cdec4b62-7244-4b80-8777-6d905a5202cd, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 92bc3f3a-c64a-4323-b745-aba7d1b54a8d, kind: Service { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a }, name: Name(\"ntp-af96c243-6c6a-4cb4-8eab-8f8a25fd030a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 160])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f85a133-0257-47cf-8244-149e630b6fde, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: e38bc018-21d0-4daa-8b2c-d199004af618, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 69cbd71e-481b-406f-9eb3-20fa7a960393, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 544a0264-7c3a-4ada-aa6e-553800c85da2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 4d9df03f-9907-4449-8d22-3a38f32b899c, kind: Service { id: 544a0264-7c3a-4ada-aa6e-553800c85da2 }, name: Name(\"external-dns-544a0264-7c3a-4ada-aa6e-553800c85da2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 114])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: f0ddfaba-8652-4628-848a-f993e079fb94, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 0ab9fb64-2e84-44e1-91ff-d472c798d3d1, kind: Service { id: f0ddfaba-8652-4628-848a-f993e079fb94 }, name: Name(\"external-dns-f0ddfaba-8652-4628-848a-f993e079fb94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 96bac47d-0e2f-4191-820f-fdc11f7f9342, kind: Service { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f }, name: Name(\"nexus-9ff5ab27-a3ab-4401-8a23-c0799de85f2f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e46afa6-9da3-407c-92bf-128b95063f04, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4ad4ab3a-b527-470c-b61b-a8987f3919b5, kind: Service { id: 8e46afa6-9da3-407c-92bf-128b95063f04 }, name: Name(\"nexus-8e46afa6-9da3-407c-92bf-128b95063f04\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ab5e3564-209c-48fa-b8f5-aa370048ac8d, kind: Service { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67 }, name: Name(\"nexus-7b52f14b-12cb-45e4-b4e3-5c517e3cba67\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 39])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: da35a429-5291-41f8-8c2f-7377bb15ed38, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5346055-3249-4357-951e-e5bf8d4dc3cf, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: be250b1e-18db-457d-9646-485f60896fb8, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f545bea-eb7a-44c2-a399-0237abf2a102, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e35814c6-9e18-4226-86a8-2ab44a0293b2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 82bbc823-811c-4ea4-adbb-453cfe570acf, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42568398-8d6b-43e8-b33e-d40f8662e422, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External } } } }, OmicronZoneConfig { id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41863173-79c0-4807-9ede-f5bf31ee3536, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68e6a369-307f-4afa-a667-cfb114baa982, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:42.639222663Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:42.639538718Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:42.639551618Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:42.664768453Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:42.664810022Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:42.686651528Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:42.68668827Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:42.707131191Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:42.707168757Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:42.72696834Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:42.727007519Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:42.74673375Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:42.746784057Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:42.765598775Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:42.76563848Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:42.785693337Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:42.785733529Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:42.804268087Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:42.804374658Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:42.825618687Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:42.825658672Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:42.844104386Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:42.844139665Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:42.864250147Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:42.864286972Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:42.884038216Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:42.88407523Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:42.904545601Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:42.904575499Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:42.926861591Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:42.926910689Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:42.947497383Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:42.947532587Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:42.969425906Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:42.969461513Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:42.990263755Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:42.990305399Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:43.012065504Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:43.012107515Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:43.032072139Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:43.032115181Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:43.052662877Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:43.052705843Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:43.052926262Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:43.053112163Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:43.158611112Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:43.158669369Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:43.158676288Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:43.158683132Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:43.158688297Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:43.15869395Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:43.158699819Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:43.15870701Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:43.158714838Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:43.15873868Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:43.158744464Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:43.158749686Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:43.15875488Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:43.158759961Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:43.158765164Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:43.158771755Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:43.158777184Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:43.158782236Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:43.15878729Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:43.15891062Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"242e185d-446b-461a-aa8d-7f1120ee82a9","remote_addr":"[fd00:1122:3344:101::1]:35174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\", TimeNotSynchronized), (\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\", TimeNotSynchronized), (\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\", TimeNotSynchronized), (\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\", TimeNotSynchronized), (\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\", TimeNotSynchronized), (\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\", TimeNotSynchronized), (\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\", TimeNotSynchronized), (\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\", TimeNotSynchronized), (\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\", TimeNotSynchronized), (\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\", TimeNotSynchronized), (\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\", TimeNotSynchronized), (\"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf\", TimeNotSynchronized), (\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\", TimeNotSynchronized), (\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\", TimeNotSynchronized), (\"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67\", TimeNotSynchronized), (\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\", TimeNotSynchronized), (\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\", TimeNotSynchronized), (\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\", TimeNotSynchronized)]","latency_us":518164,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:43.159097663Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"1.15383803s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"242e185d-446b-461a-aa8d-7f1120ee82a9\", \"content-length\": \"124\", \"date\": \"Sat, 06 Jul 2024 00:04:43 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"242e185d-446b-461a-aa8d-7f1120ee82a9\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: cdec4b62-7244-4b80-8777-6d905a5202cd, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 92bc3f3a-c64a-4323-b745-aba7d1b54a8d, kind: Service { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a }, name: Name(\"ntp-af96c243-6c6a-4cb4-8eab-8f8a25fd030a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 160])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f85a133-0257-47cf-8244-149e630b6fde, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: e38bc018-21d0-4daa-8b2c-d199004af618, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 69cbd71e-481b-406f-9eb3-20fa7a960393, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 544a0264-7c3a-4ada-aa6e-553800c85da2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 4d9df03f-9907-4449-8d22-3a38f32b899c, kind: Service { id: 544a0264-7c3a-4ada-aa6e-553800c85da2 }, name: Name(\"external-dns-544a0264-7c3a-4ada-aa6e-553800c85da2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 114])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: f0ddfaba-8652-4628-848a-f993e079fb94, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 0ab9fb64-2e84-44e1-91ff-d472c798d3d1, kind: Service { id: f0ddfaba-8652-4628-848a-f993e079fb94 }, name: Name(\"external-dns-f0ddfaba-8652-4628-848a-f993e079fb94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 96bac47d-0e2f-4191-820f-fdc11f7f9342, kind: Service { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f }, name: Name(\"nexus-9ff5ab27-a3ab-4401-8a23-c0799de85f2f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e46afa6-9da3-407c-92bf-128b95063f04, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4ad4ab3a-b527-470c-b61b-a8987f3919b5, kind: Service { id: 8e46afa6-9da3-407c-92bf-128b95063f04 }, name: Name(\"nexus-8e46afa6-9da3-407c-92bf-128b95063f04\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ab5e3564-209c-48fa-b8f5-aa370048ac8d, kind: Service { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67 }, name: Name(\"nexus-7b52f14b-12cb-45e4-b4e3-5c517e3cba67\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 39])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: da35a429-5291-41f8-8c2f-7377bb15ed38, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5346055-3249-4357-951e-e5bf8d4dc3cf, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: be250b1e-18db-457d-9646-485f60896fb8, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f545bea-eb7a-44c2-a399-0237abf2a102, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e35814c6-9e18-4226-86a8-2ab44a0293b2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 82bbc823-811c-4ea4-adbb-453cfe570acf, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42568398-8d6b-43e8-b33e-d40f8662e422, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External } } } }, OmicronZoneConfig { id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41863173-79c0-4807-9ede-f5bf31ee3536, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68e6a369-307f-4afa-a667-cfb114baa982, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:44.317830627Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:44.31814448Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:44.318152645Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:44.342309263Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:44.342352848Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:44.363769073Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:44.363814082Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:44.384129886Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:44.384179818Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:44.402733323Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:44.402772099Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:44.422953773Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:44.422996121Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:44.441987411Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:44.442045968Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:44.463022003Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:44.463068691Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:44.482031309Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:44.482075982Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:44.503253824Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:44.503297466Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:44.52403921Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:44.524079363Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:44.545453702Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:44.545507393Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:44.564013663Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:44.564053939Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:44.584682556Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:44.584729114Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:44.60338625Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:44.60342287Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:44.624561311Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:44.624602589Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:44.643620308Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:44.643664212Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:44.6621174Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:44.662163826Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:44.6805303Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:44.680567425Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:44.700249197Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:44.700286556Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:44.719786503Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:44.719842895Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:44.720031082Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:44.720157197Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:44.821416398Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:44.821467783Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:44.821477242Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:44.821482689Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:44.821489242Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:44.821494633Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:44.821499705Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:44.821506277Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:44.821511274Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:44.821516553Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:44.821521587Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:44.821528112Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:44.821533071Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:44.821538078Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:44.821561103Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:44.821566578Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:44.821571313Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:44.821576263Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:44.8215799Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:44.821664576Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"7cd32f84-98de-4e9d-a40d-ac89c848eb77","remote_addr":"[fd00:1122:3344:101::1]:35174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\", TimeNotSynchronized), (\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\", TimeNotSynchronized), (\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\", TimeNotSynchronized), (\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\", TimeNotSynchronized), (\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\", TimeNotSynchronized), (\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\", TimeNotSynchronized), (\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\", TimeNotSynchronized), (\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\", TimeNotSynchronized), (\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\", TimeNotSynchronized), (\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\", TimeNotSynchronized), (\"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67\", TimeNotSynchronized), (\"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf\", TimeNotSynchronized), (\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\", TimeNotSynchronized), (\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\", TimeNotSynchronized), (\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\", TimeNotSynchronized), (\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\", TimeNotSynchronized), (\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\", TimeNotSynchronized), (\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\", TimeNotSynchronized)]","latency_us":502365,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:44.821913171Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"2.884944562s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7cd32f84-98de-4e9d-a40d-ac89c848eb77\", \"content-length\": \"124\", \"date\": \"Sat, 06 Jul 2024 00:04:44 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7cd32f84-98de-4e9d-a40d-ac89c848eb77\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: cdec4b62-7244-4b80-8777-6d905a5202cd, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 92bc3f3a-c64a-4323-b745-aba7d1b54a8d, kind: Service { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a }, name: Name(\"ntp-af96c243-6c6a-4cb4-8eab-8f8a25fd030a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 160])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f85a133-0257-47cf-8244-149e630b6fde, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: e38bc018-21d0-4daa-8b2c-d199004af618, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 69cbd71e-481b-406f-9eb3-20fa7a960393, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 544a0264-7c3a-4ada-aa6e-553800c85da2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 4d9df03f-9907-4449-8d22-3a38f32b899c, kind: Service { id: 544a0264-7c3a-4ada-aa6e-553800c85da2 }, name: Name(\"external-dns-544a0264-7c3a-4ada-aa6e-553800c85da2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 114])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: f0ddfaba-8652-4628-848a-f993e079fb94, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 0ab9fb64-2e84-44e1-91ff-d472c798d3d1, kind: Service { id: f0ddfaba-8652-4628-848a-f993e079fb94 }, name: Name(\"external-dns-f0ddfaba-8652-4628-848a-f993e079fb94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 96bac47d-0e2f-4191-820f-fdc11f7f9342, kind: Service { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f }, name: Name(\"nexus-9ff5ab27-a3ab-4401-8a23-c0799de85f2f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e46afa6-9da3-407c-92bf-128b95063f04, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4ad4ab3a-b527-470c-b61b-a8987f3919b5, kind: Service { id: 8e46afa6-9da3-407c-92bf-128b95063f04 }, name: Name(\"nexus-8e46afa6-9da3-407c-92bf-128b95063f04\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ab5e3564-209c-48fa-b8f5-aa370048ac8d, kind: Service { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67 }, name: Name(\"nexus-7b52f14b-12cb-45e4-b4e3-5c517e3cba67\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 39])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: da35a429-5291-41f8-8c2f-7377bb15ed38, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5346055-3249-4357-951e-e5bf8d4dc3cf, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: be250b1e-18db-457d-9646-485f60896fb8, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f545bea-eb7a-44c2-a399-0237abf2a102, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e35814c6-9e18-4226-86a8-2ab44a0293b2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 82bbc823-811c-4ea4-adbb-453cfe570acf, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42568398-8d6b-43e8-b33e-d40f8662e422, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External } } } }, OmicronZoneConfig { id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41863173-79c0-4807-9ede-f5bf31ee3536, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68e6a369-307f-4afa-a667-cfb114baa982, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:47.715506938Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:47.715918964Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:47.715927252Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:47.737871681Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:47.737912734Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:47.75553682Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:47.75558032Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:47.776249069Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:47.776298008Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:47.792751944Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:47.792857731Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:47.813313264Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:47.813348993Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:47.832480697Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:47.832515836Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:47.849391508Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:47.849425783Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:47.869733484Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:47.869771255Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:47.88735792Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:47.88739289Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:47.908176118Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:47.908215372Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:47.925643066Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:47.925675906Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:47.942802802Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:47.942846874Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:47.964023283Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:47.964058767Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:47.984386115Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:47.984433497Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:48.002308547Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:48.0024158Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:48.023212652Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:48.023259416Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:48.04433152Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:48.04439364Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:48.065123993Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:48.065170663Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:48.084045927Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:48.084088471Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:48.102503724Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:48.102552541Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:48.102788687Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:48.102892545Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:48.202124589Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:48.202248126Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:48.202264326Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:48.202293698Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:48.202306701Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:48.202315945Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:48.202322939Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:48.202331592Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:48.202342731Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:48.202353699Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:48.202360412Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:48.202370977Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:48.202379884Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:48.202386774Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:48.202393402Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:48.202402094Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:48.202408338Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:48.202415116Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:48.202436397Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:48.20244521Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"e1004696-b3d1-421c-a483-e770cc4c05d5","remote_addr":"[fd00:1122:3344:101::1]:35174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\", TimeNotSynchronized), (\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\", TimeNotSynchronized), (\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\", TimeNotSynchronized), (\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\", TimeNotSynchronized), (\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\", TimeNotSynchronized), (\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\", TimeNotSynchronized), (\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\", TimeNotSynchronized), (\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\", TimeNotSynchronized), (\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\", TimeNotSynchronized), (\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\", TimeNotSynchronized), (\"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\", TimeNotSynchronized), (\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\", TimeNotSynchronized), (\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\", TimeNotSynchronized), (\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\", TimeNotSynchronized), (\"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67\", TimeNotSynchronized), (\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\", TimeNotSynchronized), (\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\", TimeNotSynchronized), (\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\", TimeNotSynchronized)]","latency_us":485389,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:48.202629909Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"1.485988808s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e1004696-b3d1-421c-a483-e770cc4c05d5\", \"content-length\": \"124\", \"date\": \"Sat, 06 Jul 2024 00:04:47 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e1004696-b3d1-421c-a483-e770cc4c05d5\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: cdec4b62-7244-4b80-8777-6d905a5202cd, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 92bc3f3a-c64a-4323-b745-aba7d1b54a8d, kind: Service { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a }, name: Name(\"ntp-af96c243-6c6a-4cb4-8eab-8f8a25fd030a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 160])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f85a133-0257-47cf-8244-149e630b6fde, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: e38bc018-21d0-4daa-8b2c-d199004af618, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 69cbd71e-481b-406f-9eb3-20fa7a960393, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 544a0264-7c3a-4ada-aa6e-553800c85da2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 4d9df03f-9907-4449-8d22-3a38f32b899c, kind: Service { id: 544a0264-7c3a-4ada-aa6e-553800c85da2 }, name: Name(\"external-dns-544a0264-7c3a-4ada-aa6e-553800c85da2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 114])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: f0ddfaba-8652-4628-848a-f993e079fb94, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 0ab9fb64-2e84-44e1-91ff-d472c798d3d1, kind: Service { id: f0ddfaba-8652-4628-848a-f993e079fb94 }, name: Name(\"external-dns-f0ddfaba-8652-4628-848a-f993e079fb94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 96bac47d-0e2f-4191-820f-fdc11f7f9342, kind: Service { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f }, name: Name(\"nexus-9ff5ab27-a3ab-4401-8a23-c0799de85f2f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e46afa6-9da3-407c-92bf-128b95063f04, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4ad4ab3a-b527-470c-b61b-a8987f3919b5, kind: Service { id: 8e46afa6-9da3-407c-92bf-128b95063f04 }, name: Name(\"nexus-8e46afa6-9da3-407c-92bf-128b95063f04\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ab5e3564-209c-48fa-b8f5-aa370048ac8d, kind: Service { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67 }, name: Name(\"nexus-7b52f14b-12cb-45e4-b4e3-5c517e3cba67\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 39])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: da35a429-5291-41f8-8c2f-7377bb15ed38, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5346055-3249-4357-951e-e5bf8d4dc3cf, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: be250b1e-18db-457d-9646-485f60896fb8, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f545bea-eb7a-44c2-a399-0237abf2a102, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e35814c6-9e18-4226-86a8-2ab44a0293b2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 82bbc823-811c-4ea4-adbb-453cfe570acf, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42568398-8d6b-43e8-b33e-d40f8662e422, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External } } } }, OmicronZoneConfig { id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41863173-79c0-4807-9ede-f5bf31ee3536, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68e6a369-307f-4afa-a667-cfb114baa982, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:49.694405873Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:49.694723241Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:49.694731257Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:49.718415651Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:49.718467767Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:49.740507531Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:49.740556834Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:49.762263015Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:49.764629442Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:49.783298777Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:49.783359294Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:49.803374075Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:49.805742527Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:49.823926358Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:49.823972418Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:49.844560842Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:49.844618039Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:49.86491074Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:49.864961357Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:49.886978891Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:49.887040889Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:49.906388249Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:49.906432688Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:49.927734496Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:49.927779272Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:49.946731099Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:49.946773568Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:49.969415855Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:49.969463302Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:49.990136284Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:49.990187247Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:50.010229259Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:50.010282482Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:50.031221155Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:50.03126865Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:50.051576439Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:50.051630149Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:50.072220141Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:50.074775087Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:50.095425224Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:50.095476132Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:50.11970637Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:50.119761262Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:50.119941171Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:50.120063817Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:50.21869003Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:50.21877629Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:50.218784653Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:50.218791965Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:50.218798171Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:50.218801838Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:50.218807144Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:50.218818807Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:50.218827563Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:50.21883286Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:50.218839338Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:50.218843313Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:50.21884666Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:50.218851994Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:50.21885731Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:50.21886077Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:50.218870558Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:50.218874908Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:50.218880401Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:50.218886055Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"c96c8e90-8d2d-4aa8-ab5c-7ad51be76a5b","remote_addr":"[fd00:1122:3344:101::1]:35174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\", TimeNotSynchronized), (\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\", TimeNotSynchronized), (\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\", TimeNotSynchronized), (\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\", TimeNotSynchronized), (\"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf\", TimeNotSynchronized), (\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\", TimeNotSynchronized), (\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\", TimeNotSynchronized), (\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\", TimeNotSynchronized), (\"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67\", TimeNotSynchronized), (\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\", TimeNotSynchronized), (\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\", TimeNotSynchronized), (\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\", TimeNotSynchronized), (\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\", TimeNotSynchronized), (\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\", TimeNotSynchronized), (\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\", TimeNotSynchronized), (\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\", TimeNotSynchronized), (\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\", TimeNotSynchronized), (\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\", TimeNotSynchronized)]","latency_us":522921,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:50.219061773Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"4.576504737s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c96c8e90-8d2d-4aa8-ab5c-7ad51be76a5b\", \"content-length\": \"124\", \"date\": \"Sat, 06 Jul 2024 00:04:49 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c96c8e90-8d2d-4aa8-ab5c-7ad51be76a5b\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:52.949988055Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:55557"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:52.950066103Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"bcc51860-20b4-4336-802f-6dec7418a6de","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:55557","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":40,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: cdec4b62-7244-4b80-8777-6d905a5202cd, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 92bc3f3a-c64a-4323-b745-aba7d1b54a8d, kind: Service { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a }, name: Name(\"ntp-af96c243-6c6a-4cb4-8eab-8f8a25fd030a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 160])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f85a133-0257-47cf-8244-149e630b6fde, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: e38bc018-21d0-4daa-8b2c-d199004af618, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 69cbd71e-481b-406f-9eb3-20fa7a960393, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 544a0264-7c3a-4ada-aa6e-553800c85da2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 4d9df03f-9907-4449-8d22-3a38f32b899c, kind: Service { id: 544a0264-7c3a-4ada-aa6e-553800c85da2 }, name: Name(\"external-dns-544a0264-7c3a-4ada-aa6e-553800c85da2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 114])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: f0ddfaba-8652-4628-848a-f993e079fb94, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 0ab9fb64-2e84-44e1-91ff-d472c798d3d1, kind: Service { id: f0ddfaba-8652-4628-848a-f993e079fb94 }, name: Name(\"external-dns-f0ddfaba-8652-4628-848a-f993e079fb94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 96bac47d-0e2f-4191-820f-fdc11f7f9342, kind: Service { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f }, name: Name(\"nexus-9ff5ab27-a3ab-4401-8a23-c0799de85f2f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e46afa6-9da3-407c-92bf-128b95063f04, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4ad4ab3a-b527-470c-b61b-a8987f3919b5, kind: Service { id: 8e46afa6-9da3-407c-92bf-128b95063f04 }, name: Name(\"nexus-8e46afa6-9da3-407c-92bf-128b95063f04\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ab5e3564-209c-48fa-b8f5-aa370048ac8d, kind: Service { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67 }, name: Name(\"nexus-7b52f14b-12cb-45e4-b4e3-5c517e3cba67\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 39])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: da35a429-5291-41f8-8c2f-7377bb15ed38, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5346055-3249-4357-951e-e5bf8d4dc3cf, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: be250b1e-18db-457d-9646-485f60896fb8, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f545bea-eb7a-44c2-a399-0237abf2a102, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e35814c6-9e18-4226-86a8-2ab44a0293b2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 82bbc823-811c-4ea4-adbb-453cfe570acf, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42568398-8d6b-43e8-b33e-d40f8662e422, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External } } } }, OmicronZoneConfig { id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41863173-79c0-4807-9ede-f5bf31ee3536, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68e6a369-307f-4afa-a667-cfb114baa982, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:54.808903932Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:54.809288669Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:54.809319889Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:54.831614635Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:54.831669498Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:54.853393351Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:54.855722337Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:54.871061347Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:54.871118479Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:54.888851156Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:54.88890707Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:54.90781858Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:54.907869403Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:54.925245232Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:54.925304455Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:54.943458756Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:54.945434723Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:54.959384264Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:54.959426855Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:54.977395701Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:54.977443824Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:54.995830196Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:54.995880588Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:55.015532699Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:55.015566384Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:55.035533029Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:55.035569696Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:55.052891107Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:55.052954286Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:55.072764505Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:55.075083458Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:55.091723882Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:55.091771208Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:55.11030857Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:55.110358296Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:55.129722239Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:55.129780852Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:55.14785801Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:55.147914373Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:55.166615796Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:55.166670734Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:55.184814125Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:55.184871642Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:55.185070254Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:55.185189441Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:55.278824666Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:55.278900746Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:55.278909333Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:55.278916214Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:55.278923367Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:55.278927792Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:55.278933952Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:55.278945042Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:55.278962453Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:55.278969642Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:55.278974809Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:55.278980715Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:55.27898604Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:55.278992725Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:55.278999456Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:55.279069132Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:55.279078132Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:55.279082154Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:55.279085905Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:55.279155516Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"308151f1-88cf-493a-9c92-5d6a4005bdf5","remote_addr":"[fd00:1122:3344:101::1]:35174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\", TimeNotSynchronized), (\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\", TimeNotSynchronized), (\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\", TimeNotSynchronized), (\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\", TimeNotSynchronized), (\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\", TimeNotSynchronized), (\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\", TimeNotSynchronized), (\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\", TimeNotSynchronized), (\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\", TimeNotSynchronized), (\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\", TimeNotSynchronized), (\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\", TimeNotSynchronized), (\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\", TimeNotSynchronized), (\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\", TimeNotSynchronized), (\"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf\", TimeNotSynchronized), (\"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\", TimeNotSynchronized), (\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\", TimeNotSynchronized), (\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\", TimeNotSynchronized), (\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\", TimeNotSynchronized), (\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\", TimeNotSynchronized)]","latency_us":468796,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:55.279330175Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"3.296838419s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"308151f1-88cf-493a-9c92-5d6a4005bdf5\", \"content-length\": \"124\", \"date\": \"Sat, 06 Jul 2024 00:04:55 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"308151f1-88cf-493a-9c92-5d6a4005bdf5\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:55.720276135Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:423","component":"producer-server","component":"MetricsManager","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:423","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"4.686124197s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: cdec4b62-7244-4b80-8777-6d905a5202cd, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 92bc3f3a-c64a-4323-b745-aba7d1b54a8d, kind: Service { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a }, name: Name(\"ntp-af96c243-6c6a-4cb4-8eab-8f8a25fd030a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 160])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f85a133-0257-47cf-8244-149e630b6fde, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: e38bc018-21d0-4daa-8b2c-d199004af618, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 69cbd71e-481b-406f-9eb3-20fa7a960393, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 544a0264-7c3a-4ada-aa6e-553800c85da2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 4d9df03f-9907-4449-8d22-3a38f32b899c, kind: Service { id: 544a0264-7c3a-4ada-aa6e-553800c85da2 }, name: Name(\"external-dns-544a0264-7c3a-4ada-aa6e-553800c85da2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 114])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: f0ddfaba-8652-4628-848a-f993e079fb94, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 0ab9fb64-2e84-44e1-91ff-d472c798d3d1, kind: Service { id: f0ddfaba-8652-4628-848a-f993e079fb94 }, name: Name(\"external-dns-f0ddfaba-8652-4628-848a-f993e079fb94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 96bac47d-0e2f-4191-820f-fdc11f7f9342, kind: Service { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f }, name: Name(\"nexus-9ff5ab27-a3ab-4401-8a23-c0799de85f2f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e46afa6-9da3-407c-92bf-128b95063f04, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4ad4ab3a-b527-470c-b61b-a8987f3919b5, kind: Service { id: 8e46afa6-9da3-407c-92bf-128b95063f04 }, name: Name(\"nexus-8e46afa6-9da3-407c-92bf-128b95063f04\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ab5e3564-209c-48fa-b8f5-aa370048ac8d, kind: Service { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67 }, name: Name(\"nexus-7b52f14b-12cb-45e4-b4e3-5c517e3cba67\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 39])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: da35a429-5291-41f8-8c2f-7377bb15ed38, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5346055-3249-4357-951e-e5bf8d4dc3cf, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: be250b1e-18db-457d-9646-485f60896fb8, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f545bea-eb7a-44c2-a399-0237abf2a102, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e35814c6-9e18-4226-86a8-2ab44a0293b2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 82bbc823-811c-4ea4-adbb-453cfe570acf, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42568398-8d6b-43e8-b33e-d40f8662e422, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External } } } }, OmicronZoneConfig { id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41863173-79c0-4807-9ede-f5bf31ee3536, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68e6a369-307f-4afa-a667-cfb114baa982, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:58.586588185Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:58.586899992Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:58.586914064Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:58.610457368Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:58.613208866Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:58.631760432Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:58.634299924Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:58.650753642Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:58.650822119Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:58.67029931Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:58.670356789Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:58.688454032Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:58.688504424Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:58.707171057Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:58.707219939Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:58.725056703Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:58.72511196Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:58.744256785Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:58.744315323Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:58.765034697Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:58.765086448Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:58.785409013Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:58.785535239Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:58.805178858Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:58.805225565Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:58.824673742Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:58.824736621Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:58.843806391Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:58.843861292Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:58.862423397Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:58.862469973Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:58.881390351Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:58.881450399Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:58.899761517Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:58.89987911Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:58.918136694Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:58.918193104Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:58.937222192Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:58.937283271Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:58.95618986Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:58.958562608Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:58.976336983Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:58.976396271Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:58.976548214Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:58.97666681Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:59.070897209Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:59.07095196Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:59.070958008Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:59.070965179Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:59.070971677Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:59.070978436Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:59.070988168Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:59.070996314Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:59.071004527Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:59.071010686Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:59.071030328Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:59.071038165Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:59.07104934Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:59.0710561Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:59.071060853Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:59.071073575Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:59.071081037Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:59.07108731Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:59.071093404Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:04:59.071145605Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"9a488d95-d114-470c-91aa-1a3f06d3209d","remote_addr":"[fd00:1122:3344:101::1]:35174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\", TimeNotSynchronized), (\"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67\", TimeNotSynchronized), (\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\", TimeNotSynchronized), (\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\", TimeNotSynchronized), (\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\", TimeNotSynchronized), (\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\", TimeNotSynchronized), (\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\", TimeNotSynchronized), (\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\", TimeNotSynchronized), (\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\", TimeNotSynchronized), (\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\", TimeNotSynchronized), (\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\", TimeNotSynchronized), (\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\", TimeNotSynchronized), (\"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\", TimeNotSynchronized), (\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\", TimeNotSynchronized), (\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\", TimeNotSynchronized), (\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\", TimeNotSynchronized), (\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\", TimeNotSynchronized), (\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\", TimeNotSynchronized)]","latency_us":483159,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:04:59.071407202Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"5.697340995s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9a488d95-d114-470c-91aa-1a3f06d3209d\", \"content-length\": \"124\", \"date\": \"Sat, 06 Jul 2024 00:04:59 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9a488d95-d114-470c-91aa-1a3f06d3209d\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: cdec4b62-7244-4b80-8777-6d905a5202cd, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 92bc3f3a-c64a-4323-b745-aba7d1b54a8d, kind: Service { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a }, name: Name(\"ntp-af96c243-6c6a-4cb4-8eab-8f8a25fd030a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 160])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f85a133-0257-47cf-8244-149e630b6fde, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: e38bc018-21d0-4daa-8b2c-d199004af618, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 69cbd71e-481b-406f-9eb3-20fa7a960393, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 544a0264-7c3a-4ada-aa6e-553800c85da2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 4d9df03f-9907-4449-8d22-3a38f32b899c, kind: Service { id: 544a0264-7c3a-4ada-aa6e-553800c85da2 }, name: Name(\"external-dns-544a0264-7c3a-4ada-aa6e-553800c85da2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 114])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: f0ddfaba-8652-4628-848a-f993e079fb94, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 0ab9fb64-2e84-44e1-91ff-d472c798d3d1, kind: Service { id: f0ddfaba-8652-4628-848a-f993e079fb94 }, name: Name(\"external-dns-f0ddfaba-8652-4628-848a-f993e079fb94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 96bac47d-0e2f-4191-820f-fdc11f7f9342, kind: Service { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f }, name: Name(\"nexus-9ff5ab27-a3ab-4401-8a23-c0799de85f2f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e46afa6-9da3-407c-92bf-128b95063f04, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4ad4ab3a-b527-470c-b61b-a8987f3919b5, kind: Service { id: 8e46afa6-9da3-407c-92bf-128b95063f04 }, name: Name(\"nexus-8e46afa6-9da3-407c-92bf-128b95063f04\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ab5e3564-209c-48fa-b8f5-aa370048ac8d, kind: Service { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67 }, name: Name(\"nexus-7b52f14b-12cb-45e4-b4e3-5c517e3cba67\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 39])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: da35a429-5291-41f8-8c2f-7377bb15ed38, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5346055-3249-4357-951e-e5bf8d4dc3cf, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: be250b1e-18db-457d-9646-485f60896fb8, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f545bea-eb7a-44c2-a399-0237abf2a102, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e35814c6-9e18-4226-86a8-2ab44a0293b2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 82bbc823-811c-4ea4-adbb-453cfe570acf, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42568398-8d6b-43e8-b33e-d40f8662e422, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External } } } }, OmicronZoneConfig { id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41863173-79c0-4807-9ede-f5bf31ee3536, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68e6a369-307f-4afa-a667-cfb114baa982, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:04.785833956Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:04.786163089Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:04.786170711Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:04.808055463Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:04.810187681Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:04.827947086Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:04.828004753Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:04.848651568Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:04.848699109Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:04.869838979Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:04.869961479Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:04.888920329Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:04.889031176Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:04.907628816Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:04.907679723Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:04.926528438Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:04.926574414Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:04.945152683Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:04.945201114Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:04.964488606Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:04.964547369Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:04.983096334Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:04.983149435Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:05.002572046Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:05.002638864Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:05.023910169Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:05.023965182Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:05.042944587Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:05.042997472Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:05.06349773Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:05.063546537Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:05.082014591Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:05.082063745Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:05.100968131Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:05.101029051Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:05.119094001Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:05.119132177Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:05.137732833Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:05.137777431Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:05.156440696Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:05.156492025Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:05.178015055Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:05.178073856Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:05.178307025Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:05.178438521Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:05.273768311Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:05.273835353Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:05.273843406Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:05.273861322Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:05.273873613Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:05.273881244Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:05.273889298Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:05.273895691Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:05.273903941Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:05.273910176Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:05.273917826Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:05.273925579Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:05.273937702Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:05.273945052Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:05.273951473Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:05.274041331Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:05.274049722Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:05.274056519Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:05.274066935Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:05.274154104Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"ace0609e-60b0-4b5c-8f10-c6e140712262","remote_addr":"[fd00:1122:3344:101::1]:35174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\", TimeNotSynchronized), (\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\", TimeNotSynchronized), (\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\", TimeNotSynchronized), (\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\", TimeNotSynchronized), (\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\", TimeNotSynchronized), (\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\", TimeNotSynchronized), (\"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf\", TimeNotSynchronized), (\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\", TimeNotSynchronized), (\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\", TimeNotSynchronized), (\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\", TimeNotSynchronized), (\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\", TimeNotSynchronized), (\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\", TimeNotSynchronized), (\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\", TimeNotSynchronized), (\"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67\", TimeNotSynchronized), (\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\", TimeNotSynchronized), (\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\", TimeNotSynchronized), (\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\", TimeNotSynchronized), (\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\", TimeNotSynchronized)]","latency_us":486874,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:05.274351152Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"6.237816419s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ace0609e-60b0-4b5c-8f10-c6e140712262\", \"content-length\": \"124\", \"date\": \"Sat, 06 Jul 2024 00:05:05 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ace0609e-60b0-4b5c-8f10-c6e140712262\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: cdec4b62-7244-4b80-8777-6d905a5202cd, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 92bc3f3a-c64a-4323-b745-aba7d1b54a8d, kind: Service { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a }, name: Name(\"ntp-af96c243-6c6a-4cb4-8eab-8f8a25fd030a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 160])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f85a133-0257-47cf-8244-149e630b6fde, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: e38bc018-21d0-4daa-8b2c-d199004af618, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 69cbd71e-481b-406f-9eb3-20fa7a960393, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 544a0264-7c3a-4ada-aa6e-553800c85da2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 4d9df03f-9907-4449-8d22-3a38f32b899c, kind: Service { id: 544a0264-7c3a-4ada-aa6e-553800c85da2 }, name: Name(\"external-dns-544a0264-7c3a-4ada-aa6e-553800c85da2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 114])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: f0ddfaba-8652-4628-848a-f993e079fb94, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 0ab9fb64-2e84-44e1-91ff-d472c798d3d1, kind: Service { id: f0ddfaba-8652-4628-848a-f993e079fb94 }, name: Name(\"external-dns-f0ddfaba-8652-4628-848a-f993e079fb94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 96bac47d-0e2f-4191-820f-fdc11f7f9342, kind: Service { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f }, name: Name(\"nexus-9ff5ab27-a3ab-4401-8a23-c0799de85f2f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e46afa6-9da3-407c-92bf-128b95063f04, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4ad4ab3a-b527-470c-b61b-a8987f3919b5, kind: Service { id: 8e46afa6-9da3-407c-92bf-128b95063f04 }, name: Name(\"nexus-8e46afa6-9da3-407c-92bf-128b95063f04\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ab5e3564-209c-48fa-b8f5-aa370048ac8d, kind: Service { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67 }, name: Name(\"nexus-7b52f14b-12cb-45e4-b4e3-5c517e3cba67\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 39])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: da35a429-5291-41f8-8c2f-7377bb15ed38, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5346055-3249-4357-951e-e5bf8d4dc3cf, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: be250b1e-18db-457d-9646-485f60896fb8, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f545bea-eb7a-44c2-a399-0237abf2a102, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e35814c6-9e18-4226-86a8-2ab44a0293b2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 82bbc823-811c-4ea4-adbb-453cfe570acf, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42568398-8d6b-43e8-b33e-d40f8662e422, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External } } } }, OmicronZoneConfig { id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41863173-79c0-4807-9ede-f5bf31ee3536, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68e6a369-307f-4afa-a667-cfb114baa982, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:11.530572531Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:11.531099409Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:11.531115262Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:11.55371651Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:11.556244989Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:11.570912238Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:11.571036527Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:11.592219354Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:11.592278811Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:11.611222084Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:11.611281804Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:11.629623617Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:11.629668346Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:11.647424854Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:11.647470793Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:11.665641367Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:11.665690118Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:11.682450972Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:11.682506698Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:11.700969186Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:11.701016259Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:11.719642399Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:11.719704603Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:11.738852781Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:11.738903754Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:11.758345764Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:11.758393952Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:11.776069037Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:11.77612675Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:11.794214699Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:11.794266356Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:11.811260566Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:11.811311229Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:11.82774256Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:11.827784373Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:11.845386787Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:11.845435969Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:11.863568963Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:11.863623995Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:11.881657033Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:11.883574711Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:11.899230404Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:11.899315634Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:11.899484283Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:11.899594235Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:11.988619272Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:11.988725052Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:11.988740783Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:11.988752699Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:11.988798018Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:11.988812418Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:11.988828422Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:11.988842963Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:11.988867001Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:11.988881185Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:11.988892239Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:11.988904989Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:11.988913577Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:11.988920373Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:11.98893378Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:11.98894609Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:11.988954921Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:11.988961418Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:11.988971506Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:11.988985634Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"35333656-d731-44ff-ba3e-4d1b2c59e22f","remote_addr":"[fd00:1122:3344:101::1]:35174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\", TimeNotSynchronized), (\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\", TimeNotSynchronized), (\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\", TimeNotSynchronized), (\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\", TimeNotSynchronized), (\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\", TimeNotSynchronized), (\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\", TimeNotSynchronized), (\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\", TimeNotSynchronized), (\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\", TimeNotSynchronized), (\"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\", TimeNotSynchronized), (\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\", TimeNotSynchronized), (\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\", TimeNotSynchronized), (\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\", TimeNotSynchronized), (\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\", TimeNotSynchronized), (\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\", TimeNotSynchronized), (\"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67\", TimeNotSynchronized), (\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\", TimeNotSynchronized), (\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\", TimeNotSynchronized), (\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\", TimeNotSynchronized)]","latency_us":457035,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:11.989256932Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"6.95871394s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"35333656-d731-44ff-ba3e-4d1b2c59e22f\", \"content-length\": \"124\", \"date\": \"Sat, 06 Jul 2024 00:05:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"35333656-d731-44ff-ba3e-4d1b2c59e22f\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:15.495817143Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:423","component":"producer-server","component":"MetricsManager","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:423","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"5.225365509s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: cdec4b62-7244-4b80-8777-6d905a5202cd, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 92bc3f3a-c64a-4323-b745-aba7d1b54a8d, kind: Service { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a }, name: Name(\"ntp-af96c243-6c6a-4cb4-8eab-8f8a25fd030a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 160])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f85a133-0257-47cf-8244-149e630b6fde, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: e38bc018-21d0-4daa-8b2c-d199004af618, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 69cbd71e-481b-406f-9eb3-20fa7a960393, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 544a0264-7c3a-4ada-aa6e-553800c85da2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 4d9df03f-9907-4449-8d22-3a38f32b899c, kind: Service { id: 544a0264-7c3a-4ada-aa6e-553800c85da2 }, name: Name(\"external-dns-544a0264-7c3a-4ada-aa6e-553800c85da2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 114])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: f0ddfaba-8652-4628-848a-f993e079fb94, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 0ab9fb64-2e84-44e1-91ff-d472c798d3d1, kind: Service { id: f0ddfaba-8652-4628-848a-f993e079fb94 }, name: Name(\"external-dns-f0ddfaba-8652-4628-848a-f993e079fb94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 96bac47d-0e2f-4191-820f-fdc11f7f9342, kind: Service { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f }, name: Name(\"nexus-9ff5ab27-a3ab-4401-8a23-c0799de85f2f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e46afa6-9da3-407c-92bf-128b95063f04, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4ad4ab3a-b527-470c-b61b-a8987f3919b5, kind: Service { id: 8e46afa6-9da3-407c-92bf-128b95063f04 }, name: Name(\"nexus-8e46afa6-9da3-407c-92bf-128b95063f04\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ab5e3564-209c-48fa-b8f5-aa370048ac8d, kind: Service { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67 }, name: Name(\"nexus-7b52f14b-12cb-45e4-b4e3-5c517e3cba67\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 39])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: da35a429-5291-41f8-8c2f-7377bb15ed38, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5346055-3249-4357-951e-e5bf8d4dc3cf, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: be250b1e-18db-457d-9646-485f60896fb8, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f545bea-eb7a-44c2-a399-0237abf2a102, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e35814c6-9e18-4226-86a8-2ab44a0293b2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 82bbc823-811c-4ea4-adbb-453cfe570acf, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42568398-8d6b-43e8-b33e-d40f8662e422, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External } } } }, OmicronZoneConfig { id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41863173-79c0-4807-9ede-f5bf31ee3536, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68e6a369-307f-4afa-a667-cfb114baa982, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:18.968572819Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:18.968890065Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:18.968897799Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:18.99334763Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:18.993405409Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:19.0179872Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:19.018109813Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:19.039749065Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:19.03980316Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:19.062643096Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:19.062721847Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:19.083601569Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:19.083655279Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:19.103786287Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:19.103839773Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:19.123981632Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:19.124034227Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:19.144278154Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:19.146741336Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:19.166084754Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:19.166135043Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:19.18691383Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:19.186971347Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:19.206698043Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:19.206749775Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:19.226982036Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:19.227030722Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:19.249287867Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:19.2493585Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:19.268971056Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:19.26901842Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:19.289601318Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:19.289655543Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:19.309913277Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:19.309962618Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:19.332679617Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:19.332736036Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:19.354244158Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:19.354303605Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:19.374611662Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:19.374677204Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:19.394898973Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:19.394965311Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:19.395226969Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:19.395373962Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:19.497423728Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:19.497496517Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:19.497505339Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:19.49753052Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:19.497548548Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:19.497556368Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:19.497579646Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:19.497587399Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:19.497593371Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:19.497598799Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:19.497611663Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:19.497617381Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:19.497622875Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:19.497630056Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:19.4976354Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:19.497642178Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:19.497648056Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:19.497654984Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:19.497661875Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:19.497742933Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"2d968726-79c5-4a09-b13a-bcffbd54f989","remote_addr":"[fd00:1122:3344:101::1]:35174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\", TimeNotSynchronized), (\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\", TimeNotSynchronized), (\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\", TimeNotSynchronized), (\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\", TimeNotSynchronized), (\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\", TimeNotSynchronized), (\"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67\", TimeNotSynchronized), (\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\", TimeNotSynchronized), (\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\", TimeNotSynchronized), (\"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf\", TimeNotSynchronized), (\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\", TimeNotSynchronized), (\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\", TimeNotSynchronized), (\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\", TimeNotSynchronized), (\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\", TimeNotSynchronized), (\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\", TimeNotSynchronized), (\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\", TimeNotSynchronized), (\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\", TimeNotSynchronized), (\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\", TimeNotSynchronized), (\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\", TimeNotSynchronized), (\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\", TimeNotSynchronized)]","latency_us":527576,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:19.497930192Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"11.596160702s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2d968726-79c5-4a09-b13a-bcffbd54f989\", \"content-length\": \"124\", \"date\": \"Sat, 06 Jul 2024 00:05:19 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2d968726-79c5-4a09-b13a-bcffbd54f989\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:23.064641816Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:48148"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:23.064738904Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"a29664a0-802c-4c8a-a33e-78e2839ff65f","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:48148","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":39,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: cdec4b62-7244-4b80-8777-6d905a5202cd, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 92bc3f3a-c64a-4323-b745-aba7d1b54a8d, kind: Service { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a }, name: Name(\"ntp-af96c243-6c6a-4cb4-8eab-8f8a25fd030a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 160])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f85a133-0257-47cf-8244-149e630b6fde, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: e38bc018-21d0-4daa-8b2c-d199004af618, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 69cbd71e-481b-406f-9eb3-20fa7a960393, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 544a0264-7c3a-4ada-aa6e-553800c85da2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 4d9df03f-9907-4449-8d22-3a38f32b899c, kind: Service { id: 544a0264-7c3a-4ada-aa6e-553800c85da2 }, name: Name(\"external-dns-544a0264-7c3a-4ada-aa6e-553800c85da2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 114])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: f0ddfaba-8652-4628-848a-f993e079fb94, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 0ab9fb64-2e84-44e1-91ff-d472c798d3d1, kind: Service { id: f0ddfaba-8652-4628-848a-f993e079fb94 }, name: Name(\"external-dns-f0ddfaba-8652-4628-848a-f993e079fb94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 96bac47d-0e2f-4191-820f-fdc11f7f9342, kind: Service { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f }, name: Name(\"nexus-9ff5ab27-a3ab-4401-8a23-c0799de85f2f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e46afa6-9da3-407c-92bf-128b95063f04, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4ad4ab3a-b527-470c-b61b-a8987f3919b5, kind: Service { id: 8e46afa6-9da3-407c-92bf-128b95063f04 }, name: Name(\"nexus-8e46afa6-9da3-407c-92bf-128b95063f04\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ab5e3564-209c-48fa-b8f5-aa370048ac8d, kind: Service { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67 }, name: Name(\"nexus-7b52f14b-12cb-45e4-b4e3-5c517e3cba67\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 39])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: da35a429-5291-41f8-8c2f-7377bb15ed38, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5346055-3249-4357-951e-e5bf8d4dc3cf, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: be250b1e-18db-457d-9646-485f60896fb8, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f545bea-eb7a-44c2-a399-0237abf2a102, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e35814c6-9e18-4226-86a8-2ab44a0293b2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 82bbc823-811c-4ea4-adbb-453cfe570acf, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42568398-8d6b-43e8-b33e-d40f8662e422, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External } } } }, OmicronZoneConfig { id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41863173-79c0-4807-9ede-f5bf31ee3536, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68e6a369-307f-4afa-a667-cfb114baa982, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:31.127418734Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:31.12767177Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:31.127685073Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:31.151570791Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:31.151638911Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:31.17192422Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:31.171998442Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:31.189677523Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:31.189728843Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:31.209911038Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:31.212349456Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:31.228140012Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:31.228218116Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:31.246681112Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:31.246733332Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:31.265567047Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:31.265614823Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:31.283454934Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:31.283502485Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:31.301449706Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:31.301504579Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:31.319789955Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:31.319830963Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:31.338572603Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:31.338611622Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:31.356705734Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:31.356759003Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:31.373851585Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:31.373892451Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:31.391945737Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:31.39199685Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:31.410839801Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:31.410881848Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:31.428124501Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:31.428179168Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:31.445682312Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:31.445742538Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:31.463788405Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:31.463839931Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:31.481231563Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:31.481276452Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:31.498967837Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:31.499024565Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:31.499285589Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:31.499430097Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:31.589545478Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:31.589632771Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:31.589642464Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:31.589652627Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:31.589662349Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:31.589670093Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:31.589678353Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:31.589709647Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:31.589719668Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:31.589737294Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:31.589744794Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:31.589754554Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:31.589764032Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:31.589773988Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:31.589783279Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:31.589788848Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:31.589796161Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:31.589801898Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:31.589808189Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:31.589908418Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"3e805ee6-dcc0-4c28-afad-ff762e31765f","remote_addr":"[fd00:1122:3344:101::1]:35174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67\", TimeNotSynchronized), (\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\", TimeNotSynchronized), (\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\", TimeNotSynchronized), (\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\", TimeNotSynchronized), (\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\", TimeNotSynchronized), (\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\", TimeNotSynchronized), (\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\", TimeNotSynchronized), (\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\", TimeNotSynchronized), (\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\", TimeNotSynchronized), (\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\", TimeNotSynchronized), (\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\", TimeNotSynchronized), (\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\", TimeNotSynchronized), (\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\", TimeNotSynchronized), (\"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf\", TimeNotSynchronized), (\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\", TimeNotSynchronized), (\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\", TimeNotSynchronized), (\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\", TimeNotSynchronized), (\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\", TimeNotSynchronized)]","latency_us":461184,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:31.590251379Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"10.352145341s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3e805ee6-dcc0-4c28-afad-ff762e31765f\", \"content-length\": \"124\", \"date\": \"Sat, 06 Jul 2024 00:05:31 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3e805ee6-dcc0-4c28-afad-ff762e31765f\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:35.814545888Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:423","component":"producer-server","component":"MetricsManager","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:423","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"14.317393873s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: cdec4b62-7244-4b80-8777-6d905a5202cd, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 92bc3f3a-c64a-4323-b745-aba7d1b54a8d, kind: Service { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a }, name: Name(\"ntp-af96c243-6c6a-4cb4-8eab-8f8a25fd030a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 160])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f85a133-0257-47cf-8244-149e630b6fde, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: e38bc018-21d0-4daa-8b2c-d199004af618, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 69cbd71e-481b-406f-9eb3-20fa7a960393, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 544a0264-7c3a-4ada-aa6e-553800c85da2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 4d9df03f-9907-4449-8d22-3a38f32b899c, kind: Service { id: 544a0264-7c3a-4ada-aa6e-553800c85da2 }, name: Name(\"external-dns-544a0264-7c3a-4ada-aa6e-553800c85da2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 114])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: f0ddfaba-8652-4628-848a-f993e079fb94, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 0ab9fb64-2e84-44e1-91ff-d472c798d3d1, kind: Service { id: f0ddfaba-8652-4628-848a-f993e079fb94 }, name: Name(\"external-dns-f0ddfaba-8652-4628-848a-f993e079fb94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 96bac47d-0e2f-4191-820f-fdc11f7f9342, kind: Service { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f }, name: Name(\"nexus-9ff5ab27-a3ab-4401-8a23-c0799de85f2f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e46afa6-9da3-407c-92bf-128b95063f04, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4ad4ab3a-b527-470c-b61b-a8987f3919b5, kind: Service { id: 8e46afa6-9da3-407c-92bf-128b95063f04 }, name: Name(\"nexus-8e46afa6-9da3-407c-92bf-128b95063f04\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ab5e3564-209c-48fa-b8f5-aa370048ac8d, kind: Service { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67 }, name: Name(\"nexus-7b52f14b-12cb-45e4-b4e3-5c517e3cba67\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 39])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: da35a429-5291-41f8-8c2f-7377bb15ed38, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5346055-3249-4357-951e-e5bf8d4dc3cf, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: be250b1e-18db-457d-9646-485f60896fb8, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f545bea-eb7a-44c2-a399-0237abf2a102, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e35814c6-9e18-4226-86a8-2ab44a0293b2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 82bbc823-811c-4ea4-adbb-453cfe570acf, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42568398-8d6b-43e8-b33e-d40f8662e422, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External } } } }, OmicronZoneConfig { id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41863173-79c0-4807-9ede-f5bf31ee3536, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68e6a369-307f-4afa-a667-cfb114baa982, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:41.971023057Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:41.971493023Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:41.971505586Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:41.994647637Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:41.994703231Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:42.016850866Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:42.016902269Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:42.038234619Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:42.038290561Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:42.057965063Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:42.05803279Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:42.079005889Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:42.079069969Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:42.099107112Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:42.099162032Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:42.118029673Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:42.118091352Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:42.136652117Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:42.136709492Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:42.155237506Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:42.155280547Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:42.173878769Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:42.173931945Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:42.193961001Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:42.194016352Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:42.213021595Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:42.213069812Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:42.23158419Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:42.231641783Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:42.249800701Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:42.249877633Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:42.26851592Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:42.268571178Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:42.287276321Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:42.287327256Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:42.306188594Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:42.306238826Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:42.324181794Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:42.324242807Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:42.341925897Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:42.342068385Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:42.36069526Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:42.360763079Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:42.361094814Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:42.361239153Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:42.456870004Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:42.456965367Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:42.456977593Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:42.457002887Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:42.457012309Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:42.457024318Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:42.457034077Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:42.457041785Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:42.45705461Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:42.457084385Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:42.457096264Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:42.457105836Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:42.457115314Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:42.457124848Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:42.457134045Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:42.457145727Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:42.457152965Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:42.45716008Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:42.457165855Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:42.457253737Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"9c5dbf28-c044-4c5f-a114-44e412144d36","remote_addr":"[fd00:1122:3344:101::1]:35174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\", TimeNotSynchronized), (\"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf\", TimeNotSynchronized), (\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\", TimeNotSynchronized), (\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\", TimeNotSynchronized), (\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\", TimeNotSynchronized), (\"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67\", TimeNotSynchronized), (\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\", TimeNotSynchronized), (\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\", TimeNotSynchronized), (\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\", TimeNotSynchronized), (\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\", TimeNotSynchronized), (\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\", TimeNotSynchronized), (\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\", TimeNotSynchronized), (\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\", TimeNotSynchronized), (\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\", TimeNotSynchronized), (\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\", TimeNotSynchronized), (\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\", TimeNotSynchronized), (\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\", TimeNotSynchronized), (\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\", TimeNotSynchronized)]","latency_us":484752,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:42.457535783Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"15.097260914s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9c5dbf28-c044-4c5f-a114-44e412144d36\", \"content-length\": \"124\", \"date\": \"Sat, 06 Jul 2024 00:05:42 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9c5dbf28-c044-4c5f-a114-44e412144d36\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:53.176919117Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:39656"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:53.177018502Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"7302154b-adbb-494d-8849-9c337585160b","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:39656","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":51,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: cdec4b62-7244-4b80-8777-6d905a5202cd, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 92bc3f3a-c64a-4323-b745-aba7d1b54a8d, kind: Service { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a }, name: Name(\"ntp-af96c243-6c6a-4cb4-8eab-8f8a25fd030a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 160])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f85a133-0257-47cf-8244-149e630b6fde, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: e38bc018-21d0-4daa-8b2c-d199004af618, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 69cbd71e-481b-406f-9eb3-20fa7a960393, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 544a0264-7c3a-4ada-aa6e-553800c85da2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 4d9df03f-9907-4449-8d22-3a38f32b899c, kind: Service { id: 544a0264-7c3a-4ada-aa6e-553800c85da2 }, name: Name(\"external-dns-544a0264-7c3a-4ada-aa6e-553800c85da2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 114])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: f0ddfaba-8652-4628-848a-f993e079fb94, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 0ab9fb64-2e84-44e1-91ff-d472c798d3d1, kind: Service { id: f0ddfaba-8652-4628-848a-f993e079fb94 }, name: Name(\"external-dns-f0ddfaba-8652-4628-848a-f993e079fb94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 96bac47d-0e2f-4191-820f-fdc11f7f9342, kind: Service { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f }, name: Name(\"nexus-9ff5ab27-a3ab-4401-8a23-c0799de85f2f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e46afa6-9da3-407c-92bf-128b95063f04, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4ad4ab3a-b527-470c-b61b-a8987f3919b5, kind: Service { id: 8e46afa6-9da3-407c-92bf-128b95063f04 }, name: Name(\"nexus-8e46afa6-9da3-407c-92bf-128b95063f04\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ab5e3564-209c-48fa-b8f5-aa370048ac8d, kind: Service { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67 }, name: Name(\"nexus-7b52f14b-12cb-45e4-b4e3-5c517e3cba67\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 39])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: da35a429-5291-41f8-8c2f-7377bb15ed38, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5346055-3249-4357-951e-e5bf8d4dc3cf, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: be250b1e-18db-457d-9646-485f60896fb8, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f545bea-eb7a-44c2-a399-0237abf2a102, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e35814c6-9e18-4226-86a8-2ab44a0293b2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 82bbc823-811c-4ea4-adbb-453cfe570acf, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42568398-8d6b-43e8-b33e-d40f8662e422, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External } } } }, OmicronZoneConfig { id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41863173-79c0-4807-9ede-f5bf31ee3536, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68e6a369-307f-4afa-a667-cfb114baa982, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:57.597277704Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:57.598433591Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:57.598441279Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:57.621155782Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:57.623934267Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:57.640971077Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:57.641038813Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:57.660597723Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:57.660655052Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:57.679191072Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:57.679259455Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:57.697563068Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:57.697634535Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:57.71735939Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:57.717417938Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:57.736647429Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:57.739007713Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:57.756144434Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:57.756201802Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:57.775351408Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:57.775399511Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:57.794487455Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:57.794543903Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:57.814354589Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:57.814402619Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:57.832701968Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:57.834666563Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:57.851896304Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:57.851960036Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:57.87203116Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:57.872079038Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:57.890579431Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:57.890629474Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:57.909875084Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:57.909922879Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:57.928886732Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:57.928942608Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:57.947773743Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:57.947834494Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:57.96631124Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:57.966370697Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:57.984631653Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:57.984711332Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:57.985046231Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:57.985254912Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:58.077118751Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:58.077231169Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:58.077242925Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:58.077254729Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:58.077262454Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:58.077281354Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:58.077304051Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:58.077322868Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:58.077334146Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:58.077341955Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:58.077375358Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:58.07738299Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:58.077392872Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:58.077465596Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:58.077480793Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:58.077488621Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:58.077494124Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:58.0774999Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:58.077507241Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:05:58.077519494Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"d01d6624-c430-44e0-bbcc-90cf11545e60","remote_addr":"[fd00:1122:3344:101::1]:35174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\", TimeNotSynchronized), (\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\", TimeNotSynchronized), (\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\", TimeNotSynchronized), (\"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67\", TimeNotSynchronized), (\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\", TimeNotSynchronized), (\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\", TimeNotSynchronized), (\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\", TimeNotSynchronized), (\"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\", TimeNotSynchronized), (\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\", TimeNotSynchronized), (\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\", TimeNotSynchronized), (\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\", TimeNotSynchronized), (\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\", TimeNotSynchronized), (\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\", TimeNotSynchronized), (\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\", TimeNotSynchronized), (\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\", TimeNotSynchronized), (\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\", TimeNotSynchronized), (\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\", TimeNotSynchronized), (\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\", TimeNotSynchronized)]","latency_us":478764,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:05:58.078228105Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"9.889847171s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d01d6624-c430-44e0-bbcc-90cf11545e60\", \"content-length\": \"124\", \"date\": \"Sat, 06 Jul 2024 00:05:57 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d01d6624-c430-44e0-bbcc-90cf11545e60\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:05.248518736Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:423","component":"producer-server","component":"MetricsManager","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:423","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"39.440867907s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: cdec4b62-7244-4b80-8777-6d905a5202cd, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 92bc3f3a-c64a-4323-b745-aba7d1b54a8d, kind: Service { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a }, name: Name(\"ntp-af96c243-6c6a-4cb4-8eab-8f8a25fd030a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 160])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f85a133-0257-47cf-8244-149e630b6fde, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: e38bc018-21d0-4daa-8b2c-d199004af618, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 69cbd71e-481b-406f-9eb3-20fa7a960393, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 544a0264-7c3a-4ada-aa6e-553800c85da2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 4d9df03f-9907-4449-8d22-3a38f32b899c, kind: Service { id: 544a0264-7c3a-4ada-aa6e-553800c85da2 }, name: Name(\"external-dns-544a0264-7c3a-4ada-aa6e-553800c85da2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 114])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: f0ddfaba-8652-4628-848a-f993e079fb94, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 0ab9fb64-2e84-44e1-91ff-d472c798d3d1, kind: Service { id: f0ddfaba-8652-4628-848a-f993e079fb94 }, name: Name(\"external-dns-f0ddfaba-8652-4628-848a-f993e079fb94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 96bac47d-0e2f-4191-820f-fdc11f7f9342, kind: Service { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f }, name: Name(\"nexus-9ff5ab27-a3ab-4401-8a23-c0799de85f2f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e46afa6-9da3-407c-92bf-128b95063f04, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4ad4ab3a-b527-470c-b61b-a8987f3919b5, kind: Service { id: 8e46afa6-9da3-407c-92bf-128b95063f04 }, name: Name(\"nexus-8e46afa6-9da3-407c-92bf-128b95063f04\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ab5e3564-209c-48fa-b8f5-aa370048ac8d, kind: Service { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67 }, name: Name(\"nexus-7b52f14b-12cb-45e4-b4e3-5c517e3cba67\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 39])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: da35a429-5291-41f8-8c2f-7377bb15ed38, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5346055-3249-4357-951e-e5bf8d4dc3cf, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: be250b1e-18db-457d-9646-485f60896fb8, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f545bea-eb7a-44c2-a399-0237abf2a102, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e35814c6-9e18-4226-86a8-2ab44a0293b2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 82bbc823-811c-4ea4-adbb-453cfe570acf, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42568398-8d6b-43e8-b33e-d40f8662e422, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External } } } }, OmicronZoneConfig { id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41863173-79c0-4807-9ede-f5bf31ee3536, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68e6a369-307f-4afa-a667-cfb114baa982, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:07.996264946Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:07.996610222Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:07.996622972Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:08.018915195Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:08.021040498Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:08.040699249Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:08.040746678Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:08.060644014Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:08.062984636Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:08.080358179Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:08.080417347Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:08.099195617Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:08.099247077Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:08.117042942Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:08.117089481Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:08.135665796Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:08.135729022Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:08.154205388Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:08.15426417Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:08.17383763Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:08.173893149Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:08.192291459Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:08.192360113Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:08.210953878Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:08.211002132Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:08.232277155Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:08.232330247Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:08.251635957Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:08.251693614Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:08.270044008Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:08.270105312Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:08.288872134Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:08.288942054Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:08.307626707Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:08.307684007Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:08.326409018Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:08.326472188Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:08.344643895Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:08.344703933Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:08.364336023Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:08.36439234Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:08.382873632Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:08.382938386Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:08.383318786Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:08.383468114Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:08.476836847Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:08.476894092Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:08.476908314Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:08.476914136Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:08.476919751Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:08.476926895Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:08.47693223Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:08.476944811Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:08.476950671Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:08.476959099Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:08.476964621Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:08.476970209Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:08.476992624Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:08.476998202Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:08.477005253Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:08.477010887Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:08.47701611Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:08.477022916Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:08.477028297Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:08.477149161Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"dd1b7f5c-07c0-4b82-b235-64c71c1eea56","remote_addr":"[fd00:1122:3344:101::1]:35174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\", TimeNotSynchronized), (\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\", TimeNotSynchronized), (\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\", TimeNotSynchronized), (\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\", TimeNotSynchronized), (\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\", TimeNotSynchronized), (\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\", TimeNotSynchronized), (\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\", TimeNotSynchronized), (\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\", TimeNotSynchronized), (\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\", TimeNotSynchronized), (\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\", TimeNotSynchronized), (\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\", TimeNotSynchronized), (\"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf\", TimeNotSynchronized), (\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\", TimeNotSynchronized), (\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\", TimeNotSynchronized), (\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\", TimeNotSynchronized), (\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\", TimeNotSynchronized), (\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\", TimeNotSynchronized), (\"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67\", TimeNotSynchronized)]","latency_us":479444,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:08.477358142Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"16.385139632s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"dd1b7f5c-07c0-4b82-b235-64c71c1eea56\", \"content-length\": \"124\", \"date\": \"Sat, 06 Jul 2024 00:06:07 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"dd1b7f5c-07c0-4b82-b235-64c71c1eea56\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:23.291727756Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:50322"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:23.29181788Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"4a89fdbd-24cf-4cf9-9a11-2785b8923f62","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:50322","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":52,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: cdec4b62-7244-4b80-8777-6d905a5202cd, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 92bc3f3a-c64a-4323-b745-aba7d1b54a8d, kind: Service { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a }, name: Name(\"ntp-af96c243-6c6a-4cb4-8eab-8f8a25fd030a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 160])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f85a133-0257-47cf-8244-149e630b6fde, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: e38bc018-21d0-4daa-8b2c-d199004af618, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 69cbd71e-481b-406f-9eb3-20fa7a960393, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 544a0264-7c3a-4ada-aa6e-553800c85da2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 4d9df03f-9907-4449-8d22-3a38f32b899c, kind: Service { id: 544a0264-7c3a-4ada-aa6e-553800c85da2 }, name: Name(\"external-dns-544a0264-7c3a-4ada-aa6e-553800c85da2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 114])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: f0ddfaba-8652-4628-848a-f993e079fb94, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 0ab9fb64-2e84-44e1-91ff-d472c798d3d1, kind: Service { id: f0ddfaba-8652-4628-848a-f993e079fb94 }, name: Name(\"external-dns-f0ddfaba-8652-4628-848a-f993e079fb94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 96bac47d-0e2f-4191-820f-fdc11f7f9342, kind: Service { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f }, name: Name(\"nexus-9ff5ab27-a3ab-4401-8a23-c0799de85f2f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e46afa6-9da3-407c-92bf-128b95063f04, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4ad4ab3a-b527-470c-b61b-a8987f3919b5, kind: Service { id: 8e46afa6-9da3-407c-92bf-128b95063f04 }, name: Name(\"nexus-8e46afa6-9da3-407c-92bf-128b95063f04\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ab5e3564-209c-48fa-b8f5-aa370048ac8d, kind: Service { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67 }, name: Name(\"nexus-7b52f14b-12cb-45e4-b4e3-5c517e3cba67\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 39])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: da35a429-5291-41f8-8c2f-7377bb15ed38, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5346055-3249-4357-951e-e5bf8d4dc3cf, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: be250b1e-18db-457d-9646-485f60896fb8, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f545bea-eb7a-44c2-a399-0237abf2a102, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e35814c6-9e18-4226-86a8-2ab44a0293b2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 82bbc823-811c-4ea4-adbb-453cfe570acf, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42568398-8d6b-43e8-b33e-d40f8662e422, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External } } } }, OmicronZoneConfig { id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41863173-79c0-4807-9ede-f5bf31ee3536, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68e6a369-307f-4afa-a667-cfb114baa982, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:24.907880471Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:24.908276433Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:24.908284177Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:24.932585083Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:24.932632455Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:24.952631395Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:24.954988211Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:24.971744082Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:24.971791511Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:24.989649764Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:24.989712475Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:25.009474416Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:25.009546586Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:25.029307381Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:25.029373504Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:25.047633686Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:25.047686674Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:25.066016131Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:25.066073301Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:25.084085274Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:25.084131569Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:25.104265383Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:25.104314621Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:25.124868812Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:25.124957304Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:25.144339939Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:25.144379868Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:25.162488825Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:25.164878986Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:25.182089331Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:25.182153073Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:25.200409171Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:25.202826966Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:25.220949385Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:25.220997611Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:25.239774712Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:25.239916413Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:25.259107232Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:25.259162836Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:25.278325729Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:25.278381418Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:25.29748374Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:25.297557297Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:25.297819877Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:25.298040785Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:25.391428131Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:25.391512235Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:25.391523466Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:25.391538973Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:25.391561464Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:25.391568345Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:25.391575311Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:25.391584442Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:25.391599921Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:25.391612868Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:25.391621231Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:25.391629931Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:25.391636766Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:25.391652872Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:25.391681457Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:25.391690878Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:25.391698136Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:25.391704961Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:25.391715386Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:25.391798693Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"9e6ba3d9-86cb-4964-8d36-49e4fcad35e0","remote_addr":"[fd00:1122:3344:101::1]:35174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\", TimeNotSynchronized), (\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\", TimeNotSynchronized), (\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\", TimeNotSynchronized), (\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\", TimeNotSynchronized), (\"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67\", TimeNotSynchronized), (\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\", TimeNotSynchronized), (\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\", TimeNotSynchronized), (\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\", TimeNotSynchronized), (\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\", TimeNotSynchronized), (\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\", TimeNotSynchronized), (\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\", TimeNotSynchronized), (\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\", TimeNotSynchronized), (\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\", TimeNotSynchronized), (\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\", TimeNotSynchronized), (\"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf\", TimeNotSynchronized), (\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\", TimeNotSynchronized), (\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\", TimeNotSynchronized), (\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\", TimeNotSynchronized), (\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\", TimeNotSynchronized)]","latency_us":482441,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:25.391982089Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"22.017294781s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9e6ba3d9-86cb-4964-8d36-49e4fcad35e0\", \"content-length\": \"124\", \"date\": \"Sat, 06 Jul 2024 00:06:24 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9e6ba3d9-86cb-4964-8d36-49e4fcad35e0\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: cdec4b62-7244-4b80-8777-6d905a5202cd, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 92bc3f3a-c64a-4323-b745-aba7d1b54a8d, kind: Service { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a }, name: Name(\"ntp-af96c243-6c6a-4cb4-8eab-8f8a25fd030a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 160])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f85a133-0257-47cf-8244-149e630b6fde, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: e38bc018-21d0-4daa-8b2c-d199004af618, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 69cbd71e-481b-406f-9eb3-20fa7a960393, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 544a0264-7c3a-4ada-aa6e-553800c85da2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 4d9df03f-9907-4449-8d22-3a38f32b899c, kind: Service { id: 544a0264-7c3a-4ada-aa6e-553800c85da2 }, name: Name(\"external-dns-544a0264-7c3a-4ada-aa6e-553800c85da2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 114])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: f0ddfaba-8652-4628-848a-f993e079fb94, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 0ab9fb64-2e84-44e1-91ff-d472c798d3d1, kind: Service { id: f0ddfaba-8652-4628-848a-f993e079fb94 }, name: Name(\"external-dns-f0ddfaba-8652-4628-848a-f993e079fb94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 96bac47d-0e2f-4191-820f-fdc11f7f9342, kind: Service { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f }, name: Name(\"nexus-9ff5ab27-a3ab-4401-8a23-c0799de85f2f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e46afa6-9da3-407c-92bf-128b95063f04, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4ad4ab3a-b527-470c-b61b-a8987f3919b5, kind: Service { id: 8e46afa6-9da3-407c-92bf-128b95063f04 }, name: Name(\"nexus-8e46afa6-9da3-407c-92bf-128b95063f04\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ab5e3564-209c-48fa-b8f5-aa370048ac8d, kind: Service { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67 }, name: Name(\"nexus-7b52f14b-12cb-45e4-b4e3-5c517e3cba67\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 39])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: da35a429-5291-41f8-8c2f-7377bb15ed38, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5346055-3249-4357-951e-e5bf8d4dc3cf, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: be250b1e-18db-457d-9646-485f60896fb8, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f545bea-eb7a-44c2-a399-0237abf2a102, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e35814c6-9e18-4226-86a8-2ab44a0293b2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 82bbc823-811c-4ea4-adbb-453cfe570acf, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42568398-8d6b-43e8-b33e-d40f8662e422, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External } } } }, OmicronZoneConfig { id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41863173-79c0-4807-9ede-f5bf31ee3536, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68e6a369-307f-4afa-a667-cfb114baa982, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:47.470361526Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:47.470887716Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:47.470967993Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:47.494819486Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:47.494870749Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:47.515093922Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:47.515157776Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:47.534394794Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:47.536756963Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:47.555272999Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:47.55532203Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:47.573655112Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:47.573725257Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:47.593703772Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:47.593754848Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:47.611991329Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:47.612037445Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:47.631126862Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:47.631185738Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:47.650452834Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:47.650506919Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:47.672073557Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:47.672125645Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:47.693676006Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:47.693725206Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:47.7153506Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:47.715428555Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:47.736870391Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:47.736915916Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:47.755595618Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:47.755637514Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:47.776044116Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:47.776111065Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:47.796093101Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:47.796136029Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:47.814021685Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:47.814070277Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:47.833230119Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:47.833289435Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:47.851624136Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:47.851669314Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:47.870857458Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:47.870904446Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:47.871213709Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:47.871389305Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:47.966550416Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:47.966639602Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:47.966657124Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:47.966682728Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:47.966695882Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:47.966708369Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:47.966719413Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:47.966730513Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:47.966740986Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:47.966768717Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:47.966781823Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:47.966793223Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:47.966811393Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:47.966820908Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:47.966829787Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:47.96683844Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:47.966849249Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:47.966864202Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:47.966890556Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2971","zone":"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:47.966898112Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"92c1afd0-5e49-4b8a-a00e-e893cb2b549f","remote_addr":"[fd00:1122:3344:101::1]:35174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\", TimeNotSynchronized), (\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\", TimeNotSynchronized), (\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\", TimeNotSynchronized), (\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\", TimeNotSynchronized), (\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\", TimeNotSynchronized), (\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\", TimeNotSynchronized), (\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\", TimeNotSynchronized), (\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\", TimeNotSynchronized), (\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\", TimeNotSynchronized), (\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\", TimeNotSynchronized), (\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\", TimeNotSynchronized), (\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\", TimeNotSynchronized), (\"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67\", TimeNotSynchronized), (\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\", TimeNotSynchronized), (\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\", TimeNotSynchronized), (\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\", TimeNotSynchronized), (\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\", TimeNotSynchronized), (\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\", TimeNotSynchronized), (\"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf\", TimeNotSynchronized)]","latency_us":495002,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:47.96710446Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"25.316634144s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"92c1afd0-5e49-4b8a-a00e-e893cb2b549f\", \"content-length\": \"124\", \"date\": \"Sat, 06 Jul 2024 00:06:47 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"92c1afd0-5e49-4b8a-a00e-e893cb2b549f\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:53.406079128Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:40829"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:06:53.406121465Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"fcfb38f2-92b1-4a8d-ac13-792291970089","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:40829","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":29,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:06:59.873886504Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:423","component":"producer-server","component":"MetricsManager","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:423","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"81.198907771s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: cdec4b62-7244-4b80-8777-6d905a5202cd, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 92bc3f3a-c64a-4323-b745-aba7d1b54a8d, kind: Service { id: af96c243-6c6a-4cb4-8eab-8f8a25fd030a }, name: Name(\"ntp-af96c243-6c6a-4cb4-8eab-8f8a25fd030a\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 160])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f85a133-0257-47cf-8244-149e630b6fde, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: e38bc018-21d0-4daa-8b2c-d199004af618, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 69cbd71e-481b-406f-9eb3-20fa7a960393, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 544a0264-7c3a-4ada-aa6e-553800c85da2, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 4d9df03f-9907-4449-8d22-3a38f32b899c, kind: Service { id: 544a0264-7c3a-4ada-aa6e-553800c85da2 }, name: Name(\"external-dns-544a0264-7c3a-4ada-aa6e-553800c85da2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 114])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: f0ddfaba-8652-4628-848a-f993e079fb94, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 0ab9fb64-2e84-44e1-91ff-d472c798d3d1, kind: Service { id: f0ddfaba-8652-4628-848a-f993e079fb94 }, name: Name(\"external-dns-f0ddfaba-8652-4628-848a-f993e079fb94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 96bac47d-0e2f-4191-820f-fdc11f7f9342, kind: Service { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f }, name: Name(\"nexus-9ff5ab27-a3ab-4401-8a23-c0799de85f2f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8e46afa6-9da3-407c-92bf-128b95063f04, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4ad4ab3a-b527-470c-b61b-a8987f3919b5, kind: Service { id: 8e46afa6-9da3-407c-92bf-128b95063f04 }, name: Name(\"nexus-8e46afa6-9da3-407c-92bf-128b95063f04\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ab5e3564-209c-48fa-b8f5-aa370048ac8d, kind: Service { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67 }, name: Name(\"nexus-7b52f14b-12cb-45e4-b4e3-5c517e3cba67\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 39])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: da35a429-5291-41f8-8c2f-7377bb15ed38, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d5346055-3249-4357-951e-e5bf8d4dc3cf, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: be250b1e-18db-457d-9646-485f60896fb8, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f545bea-eb7a-44c2-a399-0237abf2a102, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External } } } }, OmicronZoneConfig { id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External } } } }, OmicronZoneConfig { id: e35814c6-9e18-4226-86a8-2ab44a0293b2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External } } } }, OmicronZoneConfig { id: 82bbc823-811c-4ea4-adbb-453cfe570acf, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42568398-8d6b-43e8-b33e-d40f8662e422, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External } } } }, OmicronZoneConfig { id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 41863173-79c0-4807-9ede-f5bf31ee3536, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68e6a369-307f-4afa-a667-cfb114baa982, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.33359148Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.333951595Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.333959455Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: cdec4b62-7244-4b80-8777-6d905a5202cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.357760388Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.35780852Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: bd5693ce-2405-4f5e-bcea-cf00f7435aea, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.376932827Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.377058203Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f295afe-8b02-48b5-8b08-e8e6206ee96c, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.397449979Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.39749816Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 4f5c13e8-8c8d-45df-9213-4b813bd44490, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.417932536Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.417978897Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 3f85a133-0257-47cf-8244-149e630b6fde, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.437318865Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.437381097Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e38bc018-21d0-4daa-8b2c-d199004af618, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.455551284Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.455597541Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 69cbd71e-481b-406f-9eb3-20fa7a960393, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.474179436Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.474241827Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 375efe4d-2c7a-4598-9c88-6c4c60fbd4a4, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.493740676Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.493811457Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 544a0264-7c3a-4ada-aa6e-553800c85da2, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.513960889Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.51402127Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: f0ddfaba-8652-4628-848a-f993e079fb94, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.533505665Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.533548819Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: d5346055-3249-4357-951e-e5bf8d4dc3cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.551744492Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.551813153Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: a2c1f30f-a8a6-4e74-af9a-f32003a35978, dataset_name: DatasetName { pool_name: ZpoolName { id: 9e1edfda-939b-489d-a7cb-446b4599864c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.570132566Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.572479945Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 1f545bea-eb7a-44c2-a399-0237abf2a102, dataset_name: DatasetName { pool_name: ZpoolName { id: c029673f-d8f0-4d66-9f90-dcc8d5377a3a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.588477529Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.588520769Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 9129ddf3-00d8-4d5d-bf8e-e611448e5c48, dataset_name: DatasetName { pool_name: ZpoolName { id: 187c3b21-21e5-4726-9974-0290b3e4dbd4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.606469946Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.606521697Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: e35814c6-9e18-4226-86a8-2ab44a0293b2, dataset_name: DatasetName { pool_name: ZpoolName { id: 69edb71d-4c73-4b0c-a984-e973915bfc3d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.624453143Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.624500394Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 82bbc823-811c-4ea4-adbb-453cfe570acf, dataset_name: DatasetName { pool_name: ZpoolName { id: 05665420-5e42-4534-83d1-b4b73f92c491 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.642801876Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.642846527Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 42568398-8d6b-43e8-b33e-d40f8662e422, dataset_name: DatasetName { pool_name: ZpoolName { id: 1fb8d37a-fccb-4d16-a337-6873d3def608 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.660788143Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.660842824Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 538b3ff9-de02-4699-9056-4e5b6d4a8052, dataset_name: DatasetName { pool_name: ZpoolName { id: 03b41f7a-86c6-414e-8276-ffa92bf542a8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.680330433Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.680370559Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 41863173-79c0-4807-9ede-f5bf31ee3536, dataset_name: DatasetName { pool_name: ZpoolName { id: d0e51e3d-0355-4324-9d0e-c910d0a56ce1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.698731089Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.69877921Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:853","request":"NewFilesystemRequest { dataset_id: 68e6a369-307f-4afa-a667-cfb114baa982, dataset_name: DatasetName { pool_name: ZpoolName { id: af3bec9c-e31f-4896-95d7-21d956ea9d1e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.71694724Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.717026891Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.717307026Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.717449868Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.830852598Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.831227524Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.831306435Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.832427942Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.847088757Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.847333088Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.847369613Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.883988876Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.886273341Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:13.886402503Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:14.033435112Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:14.033920221Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:14.045589766Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:14.095163368Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:14.095240749Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:14.095248959Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:14.095254639Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:14.095267199Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:14.09527398Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Configuring new Omicron zone: oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:14.095435132Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:14.10095137Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:14.106244424Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:14.110666425Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:14.115702316Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:14.120535563Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:14.125422321Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:14.266564124Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:14.270983975Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:14.275965274Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:14.280293743Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:14.285440685Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:14.290059089Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:14.294548911Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:14.300061699Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:14.305892072Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:14.311091395Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:14.316271757Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:14.321742005Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:14.327273714Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:14.333638835Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:14.340350682Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:14.348281248Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:14.354814803Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:14.361104763Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:14.367329573Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:14.373916588Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:14.380758507Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:14.444748398Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:14.50623658Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:14.565975264Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:14.570583497Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:14.570612878Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:14.758917958Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:478","port":"Port { inner: PortInner(PortData { name: \"opte1\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 195, 184]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte1\" }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:14.758970543Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1200","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:14.766634238Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:14.766661238Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:15.035408288Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:478","port":"Port { inner: PortInner(PortData { name: \"opte2\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 133, 39]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte2\" }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:15.035472619Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1200","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:15.04052883Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:15.04055021Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:15.136028572Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:478","port":"Port { inner: PortInner(PortData { name: \"opte3\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 222, 208]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte3\" }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:15.13609131Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1200","zone_type":"nexus"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:15.191757404Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:15.250156637Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:15.259299913Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:15.259325543Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:16.060788907Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:478","port":"Port { inner: PortInner(PortData { name: \"opte4\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 183, 2]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte4\" }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:16.060849008Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1200","zone_type":"external_dns"} {"msg":"no nat entry found for: 10.151.2.167","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:16.065384571Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.167","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:16.065407451Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.2.168","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:16.065721336Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.168","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:16.065737336Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.2.166","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:16.06596724Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.166","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:16.06598057Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:16.075741865Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:16.075762894Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:16.842699229Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:478","port":"Port { inner: PortInner(PortData { name: \"opte5\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 161, 114]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte5\" }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:16.842777691Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1200","zone_type":"external_dns"} {"msg":"creation of nat entry successful for: 10.151.2.167","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:16.850226795Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"creation of nat entry successful for: 10.151.2.168","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:16.871119723Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:16.878547682Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:16.878577742Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"creation of nat entry successful for: 10.151.2.166","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:16.878591742Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:16.881943846Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"no nat entry found for: 10.151.2.165","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:17.847324698Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.165","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:17.847533311Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"Configuring new Omicron zone: oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:17.851318752Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:17.8845628Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:17.906690086Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"no nat entry found for: 10.151.2.164","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:17.915942063Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.164","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:17.916011614Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.2.165","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:17.916329529Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"creation of nat entry successful for: 10.151.2.164","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:17.924910477Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:17.925206241Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:18.001412918Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Installing Omicron zone: oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:18.00153311Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:18.030211078Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:18.04109957Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:18.162783734Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:23.160185271Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:23.416137687Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:23.53787506Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:23.804220632Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:56896"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:23.825848948Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"22cb62b6-9189-40ea-968e-5e408daa29af","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:56896","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":15288,"response_code":"200"} {"msg":"Profile for oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:23.957336447Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:23.960548178Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:23.960757141Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:23.976288769Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:24.082247669Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:24.082537727Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:24.102985042Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:24.104393505Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:24.109950153Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:24.110145516Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:24.126277678Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:24.13667449Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:24.136962925Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:24.140543025Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:24.150908556Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:24.16172822Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:24.193536588Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:24.193760081Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:24.213820861Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:24.214470284Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:24.231784488Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:24.232399188Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\") failed: Property not found. retry in 36.848746ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:24.449027866Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\") failed: Property not found. retry in 56.753221ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:24.464129217Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\") failed: Property not found. retry in 40.659553ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:24.476951991Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\") failed: Property not found. retry in 43.000874ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:24.491770978Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\") failed: Property not found. retry in 47.254723ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:24.506137307Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\") failed: Property not found. retry in 56.601142ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:24.777221355Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\") failed: Property not found. retry in 54.867871ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:24.821707456Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\") failed: Property not found. retry in 70.974566ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:24.885484988Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\") failed: Property not found. retry in 145.055496ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:24.948245515Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\") failed: Property not found. retry in 84.221517ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:25.061230799Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\") failed: Property not found. retry in 45.109302ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:25.186157443Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\") failed: Property not found. retry in 68.521374ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:25.235736205Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\") failed: Property not found. retry in 56.632432ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:25.363701727Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\") failed: Property not found. retry in 29.154497ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:25.391990659Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\") failed: Property not found. retry in 121.459174ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:25.420781658Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\") failed: Property not found. retry in 39.978703ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:25.478002942Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\") failed: Property not found. retry in 52.36361ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:25.540950027Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\") failed: Property not found. retry in 64.471822ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:25.568280753Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\") failed: Property not found. retry in 247.577837ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:25.600489647Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\") failed: Property not found. retry in 220.754793ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:25.624034853Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\") failed: Property not found. retry in 143.616582ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:25.756762542Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\") failed: Property not found. retry in 53.935207ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:25.771243213Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\") failed: Property not found. retry in 169.516004ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:25.784970773Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\") failed: Property not found. retry in 177.330502ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:25.801069599Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\") failed: Property not found. retry in 139.34399ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:25.815745674Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\") failed: Property not found. retry in 132.343412ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:25.83074355Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\") failed: Property not found. retry in 76.602855ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:25.845472738Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\") failed: Property not found. retry in 92.002447ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:25.856985752Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\") failed: Property not found. retry in 237.389915ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:25.868907422Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\") failed: Property not found. retry in 565.689785ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:25.880047832Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\") failed: Property not found. retry in 466.963164ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:25.890847363Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\") failed: Property not found. retry in 291.600119ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:25.900622299Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\") failed: Property not found. retry in 455.472321ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:28.243984108Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\") failed: Property not found. retry in 485.696839ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:28.591148599Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\") failed: Property not found. retry in 1.140868256s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:28.691466641Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\") failed: Property not found. retry in 474.892643ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:28.816206712Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\") failed: Property not found. retry in 381.538986ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:29.019693391Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\") failed: Property not found. retry in 584.048724ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:29.028869247Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\") failed: Property not found. retry in 129.965181ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:29.036884105Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\") failed: Property not found. retry in 341.168566ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:29.045380942Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\") failed: Property not found. retry in 132.06728ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:29.053649813Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\") failed: Property not found. retry in 108.246952ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:29.551001602Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\") failed: Property not found. retry in 362.759128ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:29.560511954Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\") failed: Property not found. retry in 1.489125656s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:29.568784536Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\") failed: Property not found. retry in 408.268459ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:29.576057502Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\") failed: Property not found. retry in 699.676582ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:29.583153166Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\") failed: Property not found. retry in 477.271205ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:29.59094942Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\") failed: Property not found. retry in 578.942142ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:29.598295547Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\") failed: Property not found. retry in 601.88317ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:29.605858818Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\") failed: Property not found. retry in 439.063868ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:29.613083143Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\") failed: Property not found. retry in 912.308122ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:29.622679806Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\") failed: Property not found. retry in 569.775091ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:29.667433722Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\") failed: Property not found. retry in 1.147997349s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:29.842071699Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\") failed: Property not found. retry in 645.676817ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:29.932902079Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\") failed: Property not found. retry in 1.152161024s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:29.993902533Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\") failed: Property not found. retry in 718.414221ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:30.061760776Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\") failed: Property not found. retry in 1.105776989s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:30.077574659Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\") failed: Property not found. retry in 1.182416489s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:30.188156204Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\") failed: Property not found. retry in 1.318444482s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:30.217118836Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\") failed: Property not found. retry in 837.900461ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:30.246637608Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\") failed: Property not found. retry in 1.496369106s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:30.293194521Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:30.32137177Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\") failed: Property not found. retry in 1.212000227s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:30.577100093Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:30.577212395Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\") failed: Property not found. retry in 1.383389415s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:30.589745595Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\") failed: Property not found. retry in 1.387965747s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:30.790803364Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\") failed: Property not found. retry in 69.933216ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:30.86003735Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\") failed: Property not found. retry in 132.890689ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:30.938514092Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\") failed: Property not found. retry in 804.279581ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:31.000248024Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\") failed: Property not found. retry in 991.804505ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:31.068964718Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\") failed: Property not found. retry in 102.28439ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:31.082215527Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\") failed: Property not found. retry in 570.723287ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:31.093165253Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\") failed: Property not found. retry in 777.057262ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:31.155135591Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\") failed: Property not found. retry in 789.915433ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:31.192199602Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\") failed: Property not found. retry in 319.061366ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:31.200238781Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\") failed: Property not found. retry in 512.18607ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:31.495755919Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\") failed: Property not found. retry in 1.134572121s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:31.529404009Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\") failed: Property not found. retry in 608.399066ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:31.543993164Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\") failed: Property not found. retry in 1.105382617s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:31.673016529Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\") failed: Property not found. retry in 1.058605504s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:31.801048402Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\") failed: Property not found. retry in 1.020312149s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:31.809421236Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\") failed: Property not found. retry in 1.499399592s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:31.818542977Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\") failed: Property not found. retry in 1.193459702s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:31.943005949Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\") failed: Property not found. retry in 804.317884ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:31.983750289Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\") failed: Property not found. retry in 1.486470902s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:32.13412634Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\") failed: Property not found. retry in 796.266045ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:32.141928952Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\") failed: Property not found. retry in 894.299953ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:32.149308192Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\") failed: Property not found. retry in 1.019033694s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:32.161735679Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\") failed: Property not found. retry in 777.417127ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:32.187695645Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\") failed: Property not found. retry in 501.173473ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:32.675067612Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\") failed: Property not found. retry in 1.361762597s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:32.789870558Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\") failed: Property not found. retry in 1.16313063s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:32.799769816Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\") failed: Property not found. retry in 1.066909436s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:32.839320067Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\") failed: Property not found. retry in 697.055412ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:32.868548584Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\") failed: Property not found. retry in 1.240942856s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:32.946435307Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\") failed: Property not found. retry in 892.285791ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:32.972966791Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\") failed: Property not found. retry in 1.230826147s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:33.053416019Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\") failed: Property not found. retry in 779.180641ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:33.146022924Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\") failed: Property not found. retry in 508.321289ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:33.185313461Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\") failed: Property not found. retry in 926.332589ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:33.305366277Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\") failed: Property not found. retry in 1.18017822s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:33.327629822Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\") failed: Property not found. retry in 673.407515ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:33.576842861Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\") failed: Property not found. retry in 1.205003746s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:33.797726129Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\") failed: Property not found. retry in 788.686403ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:33.917363167Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\") failed: Property not found. retry in 1.171795075s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:34.03590755Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\") failed: Property not found. retry in 985.337818ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:34.184570038Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\") failed: Property not found. retry in 760.388865ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:34.374105989Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\") failed: Property not found. retry in 525.471001ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:34.470348505Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\") failed: Property not found. retry in 708.249274ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:34.532659374Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\") failed: Property not found. retry in 779.310142ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:34.595742797Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\") failed: Property not found. retry in 513.798255ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:34.657762677Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\") failed: Property not found. retry in 617.171587ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:34.687074695Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\") failed: Property not found. retry in 1.245853873s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:34.784413429Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:34.784660763Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:34.784872296Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\") failed: Property not found. retry in 1.189869147s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:34.834000551Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\") failed: Property not found. retry in 652.048957ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:34.881190459Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:34.881344737Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:34.951159261Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:35.020826823Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\") failed: Property not found. retry in 769.997224ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:35.063376093Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:35.063530585Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\") failed: Property not found. retry in 518.183871ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:35.112539058Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\") failed: Property not found. retry in 1.422616054s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:35.174895625Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\") failed: Property not found. retry in 609.484438ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:35.24988957Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\") failed: Property not found. retry in 703.358491ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:35.327342061Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\") failed: Property not found. retry in 1.015571101s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:35.47829648Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\") failed: Property not found. retry in 1.35789599s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:35.538445656Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\") failed: Property not found. retry in 1.013009178s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:35.587918686Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\") failed: Property not found. retry in 66.893723ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:35.664440148Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\") failed: Property not found. retry in 55.123439ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:35.693494401Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\") failed: Property not found. retry in 984.675527ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:35.72223712Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\") failed: Property not found. retry in 52.991781ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:35.788751542Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\") failed: Property not found. retry in 1.349155142s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:35.813226823Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\") failed: Property not found. retry in 1.034396575s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:35.840012471Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\") failed: Property not found. retry in 746.705346ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:35.945382933Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\") failed: Property not found. retry in 81.322233ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:35.985170858Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\") failed: Property not found. retry in 72.271027ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:36.022543217Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\") failed: Property not found. retry in 143.426455ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:36.058258865Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\") failed: Property not found. retry in 597.586441ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:36.105968096Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\") failed: Property not found. retry in 702.638088ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:36.153957552Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\") failed: Property not found. retry in 162.963811ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:36.204914116Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\") failed: Property not found. retry in 249.896395ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:36.240465184Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\") failed: Property not found. retry in 501.954765ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:36.304871622Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\") failed: Property not found. retry in 883.985628ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:36.338252274Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\") failed: Property not found. retry in 196.333157ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:36.370952406Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\") failed: Property not found. retry in 216.085623ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:36.40411626Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\") failed: Property not found. retry in 591.364175ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:36.566144255Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\") failed: Property not found. retry in 695.006648ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:36.612483602Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\") failed: Property not found. retry in 861.156409ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:36.685621109Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\") failed: Property not found. retry in 798.146892ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:36.718031806Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\") failed: Property not found. retry in 580.590869ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:36.726143466Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\") failed: Property not found. retry in 562.220249ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:36.73390647Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\") failed: Property not found. retry in 1.026229476s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:36.742043633Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\") failed: Property not found. retry in 1.101648235s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:36.749236105Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\") failed: Property not found. retry in 610.551561ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:36.756479271Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\") failed: Property not found. retry in 787.958296ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:36.815865498Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\") failed: Property not found. retry in 1.490784297s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:36.866249433Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\") failed: Property not found. retry in 564.040684ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:36.927611623Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\") failed: Property not found. retry in 775.538413ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:36.935685042Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\") failed: Property not found. retry in 1.119024473s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:37.167571373Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\") failed: Property not found. retry in 1.45554182s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:37.174701377Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\") failed: Property not found. retry in 1.127443299s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:37.230544779Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\") failed: Property not found. retry in 532.374851ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:37.305355613Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\") failed: Property not found. retry in 471.280166ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:37.314465628Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\") failed: Property not found. retry in 1.048627341s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:37.322896472Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\") failed: Property not found. retry in 872.997354ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:37.375307419Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\") failed: Property not found. retry in 863.182146ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:37.500830103Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\") failed: Property not found. retry in 728.448768ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:37.525989926Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\") failed: Property not found. retry in 758.210196ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:37.558770298Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\") failed: Property not found. retry in 880.235734ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:37.614243808Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\") failed: Property not found. retry in 1.110832023s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:37.722200019Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\") failed: Property not found. retry in 1.408696648s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:37.780857543Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\") failed: Property not found. retry in 524.962894ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:37.796238379Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\") failed: Property not found. retry in 890.239693ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:37.849281854Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\") failed: Property not found. retry in 613.641776ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:37.861091604Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\") failed: Property not found. retry in 1.474761828s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:38.302522032Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\") failed: Property not found. retry in 1.168290421s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:38.311811829Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\") failed: Property not found. retry in 1.45079524s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:38.320709625Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\") failed: Property not found. retry in 1.413169561s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:38.329545417Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\") failed: Property not found. retry in 865.413919ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:38.337192835Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:38.337440539Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:38.337632751Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:38.337665022Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:38.341873439Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\") failed: Property not found. retry in 663.137344ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:38.367677281Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\") failed: Property not found. retry in 1.292878092s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:38.379905397Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\") failed: Property not found. retry in 876.452058ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:38.391297578Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\") failed: Property not found. retry in 1.437970817s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:38.400347063Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\") failed: Property not found. retry in 618.774432ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:38.485644665Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\") failed: Property not found. retry in 53.494165ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:38.501929549Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\") failed: Property not found. retry in 802.391073ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:38.509751599Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\") failed: Property not found. retry in 50.041802ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:38.51666314Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\") failed: Property not found. retry in 117.265123ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:38.562708265Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\") failed: Property not found. retry in 79.313048ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:38.575854755Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\") failed: Property not found. retry in 796.38398ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:38.63898556Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\") failed: Property not found. retry in 292.862599ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:38.663585675Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\") failed: Property not found. retry in 202.80897ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:38.73090413Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\") failed: Property not found. retry in 932.922946ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:38.750026755Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\") failed: Property not found. retry in 971.765039ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:38.843361035Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\") failed: Property not found. retry in 266.459041ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:38.945030419Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\") failed: Property not found. retry in 265.402237ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:38.969593671Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\") failed: Property not found. retry in 754.480592ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:39.04111112Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\") failed: Property not found. retry in 549.063905ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:39.115209135Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\") failed: Property not found. retry in 996.796242ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:39.19945913Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\") failed: Property not found. retry in 639.643528ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:39.212079906Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\") failed: Property not found. retry in 1.001220079s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:39.219640003Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\") failed: Property not found. retry in 431.764303ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:39.24263197Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\") failed: Property not found. retry in 1.282708279s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:39.277275212Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\") failed: Property not found. retry in 1.435194826s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:39.321656971Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\") failed: Property not found. retry in 873.179991ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:39.445423787Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\") failed: Property not found. retry in 891.82881ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:39.490699598Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\") failed: Property not found. retry in 852.601337ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:39.673902735Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\") failed: Property not found. retry in 713.408072ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:39.682811477Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\") failed: Property not found. retry in 603.445316ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:39.691747224Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\") failed: Property not found. retry in 892.576818ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:39.699831998Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\") failed: Property not found. retry in 1.367813331s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:39.75191724Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\") failed: Property not found. retry in 1.462974479s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:39.780183172Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\") failed: Property not found. retry in 783.178575ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:39.787750632Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\") failed: Property not found. retry in 537.040393ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:39.803735247Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\") failed: Property not found. retry in 641.156763ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:39.823634823Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\") failed: Property not found. retry in 753.254886ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:39.847467795Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\") failed: Property not found. retry in 1.123171683s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:39.859827903Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\") failed: Property not found. retry in 814.430151ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:40.206000009Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\") failed: Property not found. retry in 1.278874641s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:40.230406859Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\") failed: Property not found. retry in 732.467691ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:40.304829847Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\") failed: Property not found. retry in 1.138386239s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:40.329233415Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\") failed: Property not found. retry in 711.33652ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:40.349581161Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\") failed: Property not found. retry in 1.132392918s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:40.391861796Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\") failed: Property not found. retry in 1.427155153s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:40.405626865Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\") failed: Property not found. retry in 875.453007ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:40.473734843Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\") failed: Property not found. retry in 1.283870759s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:40.536313442Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\") failed: Property not found. retry in 1.370568068s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:40.569682275Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\") failed: Property not found. retry in 1.446169559s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:40.580556728Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\") failed: Property not found. retry in 1.109228897s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:40.601798047Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\") failed: Property not found. retry in 783.937681ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:40.60946906Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\") failed: Property not found. retry in 1.113184577s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:40.767225138Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\") failed: Property not found. retry in 536.744642ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:40.992894211Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\") failed: Property not found. retry in 818.542642ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:41.030806416Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\") failed: Property not found. retry in 690.745358ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:41.047977251Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\") failed: Property not found. retry in 1.066193852s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:41.069004788Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\") failed: Property not found. retry in 1.105624689s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:41.128517136Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\") failed: Property not found. retry in 1.1368734s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:41.253877618Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\") failed: Property not found. retry in 1.030381924s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:41.361774379Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\") failed: Property not found. retry in 1.496671564s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:41.403885952Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\") failed: Property not found. retry in 540.751535ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:41.47825613Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\") failed: Property not found. retry in 562.253594ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:41.523413321Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\") failed: Property not found. retry in 874.87464ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:41.533904696Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\") failed: Property not found. retry in 1.131293068s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:41.541610601Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\") failed: Property not found. retry in 1.185167974s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:41.727556329Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\") failed: Property not found. retry in 1.44232605s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:41.752855264Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:41.794393577Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:41.794892115Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\") failed: Property not found. retry in 1.200681866s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:41.851487548Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\") failed: Property not found. retry in 1.49633958s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:41.861881631Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\") failed: Property not found. retry in 550.923456ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:41.873809764Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\") failed: Property not found. retry in 1.070634952s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:41.891469937Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\") failed: Property not found. retry in 1.371589726s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:41.950234299Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67\") failed: Property not found. retry in 59.457167ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:41.999245263Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\") failed: Property not found. retry in 1.127140027s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:42.029055269Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\") failed: Property not found. retry in 1.25502631s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:42.03636666Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67\") failed: Property not found. retry in 92.415328ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:42.066859946Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\") failed: Property not found. retry in 1.149458156s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:42.095710417Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\") failed: Property not found. retry in 1.178387742s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:42.144497596Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67\") failed: Property not found. retry in 299.632355ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:42.168393047Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\") failed: Property not found. retry in 1.149243773s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:42.242902177Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\") failed: Property not found. retry in 1.446097942s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:42.402365662Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\") failed: Property not found. retry in 669.987183ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:42.411627643Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\") failed: Property not found. retry in 601.065084ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:42.419286842Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\") failed: Property not found. retry in 1.149845169s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:42.43356934Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67\") failed: Property not found. retry in 511.362277ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:42.477477221Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\") failed: Property not found. retry in 1.113990971s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:42.684546357Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\") failed: Property not found. retry in 545.366706ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:42.910786236Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\") failed: Property not found. retry in 913.674513ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:42.922679899Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\") failed: Property not found. retry in 561.849849ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:42.972172799Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67\") failed: Property not found. retry in 1.017186947s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:42.998484759Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\") failed: Property not found. retry in 1.069856292s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:43.028634145Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\") failed: Property not found. retry in 536.613888ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:43.06116471Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\") failed: Property not found. retry in 776.030267ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:43.09129808Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\") failed: Property not found. retry in 1.267434854s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:43.165404442Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\") failed: Property not found. retry in 888.675584ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:43.204231265Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\") failed: Property not found. retry in 761.407313ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:43.254300563Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\") failed: Property not found. retry in 1.281264526s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:43.301180591Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\") failed: Property not found. retry in 770.34526ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:43.330760533Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\") failed: Property not found. retry in 1.105680054s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:43.338508777Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\") failed: Property not found. retry in 828.301632ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:43.366346237Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\") failed: Property not found. retry in 1.007372616s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:43.401097375Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\") failed: Property not found. retry in 1.090079565s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:43.464720712Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\") failed: Property not found. retry in 1.204617677s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:43.543283536Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\") failed: Property not found. retry in 1.084411454s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:43.59426682Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\") failed: Property not found. retry in 955.497802ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:43.611118589Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\") failed: Property not found. retry in 1.477174472s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:43.813676333Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\") failed: Property not found. retry in 939.991565ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:43.849683408Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\") failed: Property not found. retry in 621.167638ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:43.8616426Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\") failed: Property not found. retry in 724.725731ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:43.879346631Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\") failed: Property not found. retry in 1.295247833s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:44.029446628Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67\") failed: Property not found. retry in 1.162409718s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:44.041998358Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\") failed: Property not found. retry in 604.327935ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:44.107103598Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\") failed: Property not found. retry in 1.069582099s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:44.120629619Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\") failed: Property not found. retry in 1.383198868s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:44.242289515Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\") failed: Property not found. retry in 613.717622ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:44.256888839Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\") failed: Property not found. retry in 837.690351ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:44.422288458Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\") failed: Property not found. retry in 1.448107334s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:44.447181206Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\") failed: Property not found. retry in 796.080776ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:44.460427837Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\") failed: Property not found. retry in 801.041277ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:44.495249043Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\") failed: Property not found. retry in 1.094812281s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:44.567614848Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\") failed: Property not found. retry in 520.357499ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:44.57961345Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\") failed: Property not found. retry in 1.438409087s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:44.593557133Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\") failed: Property not found. retry in 1.285063879s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:44.615541996Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\") failed: Property not found. retry in 820.240034ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:44.690330919Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\") failed: Property not found. retry in 1.01011614s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:44.722672504Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\") failed: Property not found. retry in 1.317963136s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:44.7600256Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\") failed: Property not found. retry in 633.546772ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:44.800272203Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\") failed: Property not found. retry in 866.508676ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:44.881387763Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\") failed: Property not found. retry in 630.039697ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:45.114302726Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\") failed: Property not found. retry in 1.078126982s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:45.203317027Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67\") failed: Property not found. retry in 661.759062ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:45.215996359Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\") failed: Property not found. retry in 930.109541ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:45.270209195Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\") failed: Property not found. retry in 968.802838ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:45.281469625Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\") failed: Property not found. retry in 511.167019ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:45.305835324Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\") failed: Property not found. retry in 1.090729636s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:45.318697098Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\") failed: Property not found. retry in 737.898115ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:45.339075623Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\") failed: Property not found. retry in 1.39915977s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:45.447535815Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\") failed: Property not found. retry in 1.102079767s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:45.523795852Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\") failed: Property not found. retry in 1.002770013s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:45.638847239Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\") failed: Property not found. retry in 1.22776949s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:45.676772065Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\") failed: Property not found. retry in 1.36118293s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:45.746453557Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\") failed: Property not found. retry in 601.275703ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:45.757667066Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\") failed: Property not found. retry in 1.031374514s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:45.769052318Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\") failed: Property not found. retry in 1.227617884s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:45.8405525Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67\") failed: Property not found. retry in 1.143107529s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:45.890100605Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\") failed: Property not found. retry in 926.392895ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:45.908658707Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\") failed: Property not found. retry in 1.41402067s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:45.921044914Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\") failed: Property not found. retry in 693.212109ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:46.045910605Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\") failed: Property not found. retry in 920.099533ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:46.094508834Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\") failed: Property not found. retry in 824.837265ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:46.108143701Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:46.108927864Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:46.109244709Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\") failed: Property not found. retry in 1.130736751s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:46.212737941Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\") failed: Property not found. retry in 1.45764721s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:46.265837149Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\") failed: Property not found. retry in 1.109192571s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:46.292806869Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf\") failed: Property not found. retry in 31.549143ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:46.322828278Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf\") failed: Property not found. retry in 138.375822ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:46.366393194Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\") failed: Property not found. retry in 1.290539847s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:46.375034038Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\") failed: Property not found. retry in 1.182259387s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:46.434926997Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf\") failed: Property not found. retry in 248.191995ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:46.515326891Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\") failed: Property not found. retry in 666.438874ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:46.636305083Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\") failed: Property not found. retry in 1.492842413s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:46.651990273Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\") failed: Property not found. retry in 1.051413746s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:46.749080833Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf\") failed: Property not found. retry in 219.799952ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:46.776143829Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\") failed: Property not found. retry in 1.429672703s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:46.809944344Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\") failed: Property not found. retry in 997.316579ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:46.846300325Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\") failed: Property not found. retry in 518.380105ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:46.856925805Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\") failed: Property not found. retry in 1.06072667s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:46.917003984Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\") failed: Property not found. retry in 1.079639459s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:46.943710908Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf\") failed: Property not found. retry in 1.060114765s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:47.006468596Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\") failed: Property not found. retry in 1.385809815s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:47.026186945Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67\") failed: Property not found. retry in 968.506254ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:47.044172024Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\") failed: Property not found. retry in 1.473438115s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:47.07897125Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\") failed: Property not found. retry in 707.907075ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:47.11907352Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\") failed: Property not found. retry in 1.033963611s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:47.315473461Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\") failed: Property not found. retry in 1.132999408s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:47.346085413Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\") failed: Property not found. retry in 878.693717ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:47.357999053Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\") failed: Property not found. retry in 1.352276267s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:47.390481622Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\") failed: Property not found. retry in 924.351546ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:47.417421592Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\") failed: Property not found. retry in 1.473558913s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:47.630139788Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\") failed: Property not found. retry in 1.229937234s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:47.681588739Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\") failed: Property not found. retry in 775.710398ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:47.737223567Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\") failed: Property not found. retry in 708.00548ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:47.816093717Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\") failed: Property not found. retry in 1.050222287s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:47.839098081Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\") failed: Property not found. retry in 532.907492ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:47.856832517Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\") failed: Property not found. retry in 624.587884ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:47.990957678Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67\") failed: Property not found. retry in 1.08808756s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:48.027801866Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\") failed: Property not found. retry in 590.770792ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:48.043120791Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf\") failed: Property not found. retry in 1.040533679s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:48.095783512Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\") failed: Property not found. retry in 704.449901ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:48.160347913Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\") failed: Property not found. retry in 1.374469633s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:48.256742602Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\") failed: Property not found. retry in 1.163868186s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:48.272890869Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\") failed: Property not found. retry in 727.921263ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:48.358075139Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\") failed: Property not found. retry in 1.0098691s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:48.372863685Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\") failed: Property not found. retry in 848.268885ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:48.404785135Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\") failed: Property not found. retry in 665.133277ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:48.427509425Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\") failed: Property not found. retry in 1.164182089s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:48.495249379Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\") failed: Property not found. retry in 539.649179ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:48.529378034Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\") failed: Property not found. retry in 1.235275355s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:48.546352915Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\") failed: Property not found. retry in 1.359001657s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:48.569628536Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\") failed: Property not found. retry in 1.326146811s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:48.629861568Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\") failed: Property not found. retry in 1.419314804s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:48.648835981Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\") failed: Property not found. retry in 707.461766ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:48.75835972Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\") failed: Property not found. retry in 850.715933ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:48.881593447Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\") failed: Property not found. retry in 727.573389ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:48.905174023Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\") failed: Property not found. retry in 737.445459ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:48.930726911Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\") failed: Property not found. retry in 1.357118543s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:49.083310197Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\") failed: Property not found. retry in 956.048044ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:49.099815281Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\") failed: Property not found. retry in 1.230846432s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:49.110609283Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\") failed: Property not found. retry in 1.28003573s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:49.12391593Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67\") failed: Property not found. retry in 804.77121ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:49.135555771Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf\") failed: Property not found. retry in 880.889347ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:49.147183547Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\") failed: Property not found. retry in 762.902314ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:49.269036502Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\") failed: Property not found. retry in 1.383681598s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:49.396308273Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\") failed: Property not found. retry in 771.099386ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:49.448958944Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\") failed: Property not found. retry in 1.145482484s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:49.478468545Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\") failed: Property not found. retry in 1.219345554s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:49.644036558Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\") failed: Property not found. retry in 1.07522624s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:49.654063399Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\") failed: Property not found. retry in 1.495959217s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:49.671753826Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\") failed: Property not found. retry in 596.378987ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:49.682132536Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\") failed: Property not found. retry in 739.879358ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:49.742290477Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\") failed: Property not found. retry in 1.468021197s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:49.792539719Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\") failed: Property not found. retry in 507.316103ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:49.940492521Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67\") failed: Property not found. retry in 879.01812ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:49.952021755Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\") failed: Property not found. retry in 594.490381ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:49.968222054Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf\") failed: Property not found. retry in 1.448923745s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:50.039893335Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\") failed: Property not found. retry in 901.870951ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:50.050280514Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\") failed: Property not found. retry in 635.397939ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:50.067717022Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\") failed: Property not found. retry in 1.415720245s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:50.079339265Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\") failed: Property not found. retry in 1.376233086s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:50.235678617Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\") failed: Property not found. retry in 1.144188345s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:50.293293753Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\") failed: Property not found. retry in 751.933416ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:50.354558311Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\") failed: Property not found. retry in 796.291499ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:50.418708286Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\") failed: Property not found. retry in 734.898641ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:50.452561646Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\") failed: Property not found. retry in 986.533692ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:50.463748795Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\") failed: Property not found. retry in 917.228042ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:50.497644276Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\") failed: Property not found. retry in 657.548341ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:50.577981674Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\") failed: Property not found. retry in 1.341023767s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:50.635964264Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\") failed: Property not found. retry in 1.01445716s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:50.718555842Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\") failed: Property not found. retry in 701.593631ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:50.742270411Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\") failed: Property not found. retry in 669.221983ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:50.795223066Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67\") failed: Property not found. retry in 1.38157637s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:50.844186358Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\") failed: Property not found. retry in 1.25440408s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:50.875104732Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\") failed: Property not found. retry in 727.540072ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:50.966074124Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\") failed: Property not found. retry in 1.332087679s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:51.120019442Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\") failed: Property not found. retry in 887.493895ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:51.178704919Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\") failed: Property not found. retry in 1.287436646s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:51.201447754Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\") failed: Property not found. retry in 644.579545ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:51.226693755Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\") failed: Property not found. retry in 1.433706294s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:51.25334393Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\") failed: Property not found. retry in 1.042874412s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:51.273347899Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c\") failed: Property not found. retry in 1.07774047s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:51.437231565Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\") failed: Property not found. retry in 972.443548ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:51.451437212Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\") failed: Property not found. retry in 1.301598063s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:51.462081032Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38\") failed: Property not found. retry in 1.483416576s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:51.474571131Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\") failed: Property not found. retry in 1.140413102s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:51.486009074Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf\") failed: Property not found. retry in 529.186101ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:51.504500537Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\") failed: Property not found. retry in 1.249588398s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:51.517601108Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\") failed: Property not found. retry in 922.165713ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:51.628193294Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\") failed: Property not found. retry in 855.039349ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:51.709112815Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2\") failed: Property not found. retry in 1.409334352s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:51.745954064Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\") failed: Property not found. retry in 1.210233379s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:51.887412562Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536\") failed: Property not found. retry in 1.170821635s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:51.994468019Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf\") failed: Property not found. retry in 1.083251075s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:52.048337996Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\") failed: Property not found. retry in 568.697094ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:52.080071348Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\") failed: Property not found. retry in 633.774011ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:52.143807965Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67\") failed: Property not found. retry in 1.466590342s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:52.238404105Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\") failed: Property not found. retry in 559.852615ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:52.327777933Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e\") failed: Property not found. retry in 712.693796ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:52.435743205Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\") failed: Property not found. retry in 1.025495698s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:52.462466122Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8\") failed: Property not found. retry in 1.376258687s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:52.501063958Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\") failed: Property not found. retry in 690.318415ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:52.562227372Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\") failed: Property not found. retry in 522.995772ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:52.575375495Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422\") failed: Property not found. retry in 790.382904ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:52.638332109Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48\") failed: Property not found. retry in 1.425879148s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:52.658736265Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052\") failed: Property not found. retry in 763.798774ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:52.697336691Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f\") failed: Property not found. retry in 1.3359816s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:52.775809844Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2\") failed: Property not found. retry in 651.740623ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:52.784720786Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102\") failed: Property not found. retry in 828.662695ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:52.794401491Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978\") failed: Property not found. retry in 1.091151121s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:52.900195177Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94\") failed: Property not found. retry in 1.210178726s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:53.109256307Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982\") failed: Property not found. retry in 995.224683ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:53.118423574Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf\") failed: Property not found. retry in 720.079802ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:53.142995326Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf\") failed: Property not found. retry in 633.278006ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:53.268730652Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04\") failed: Property not found. retry in 751.028876ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:53.506942947Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf\") failed: Property not found. retry in 1.489676144s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:07:53.88174612Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:53.934679735Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:58897"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:53.940007558Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"cbe2bfd1-8171-433c-b05f-8e20c2dd0e3d","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:58897","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":1111,"response_code":"200"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:55.429264709Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_crucible_538b3ff9-de02-4699-9056-4e5b6d4a8052"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:55.429485277Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_crucible_42568398-8d6b-43e8-b33e-d40f8662e422"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:55.429494212Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_nexus_8e46afa6-9da3-407c-92bf-128b95063f04"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:55.429507431Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_nexus_7b52f14b-12cb-45e4-b4e3-5c517e3cba67"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:55.429513759Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_crucible_pantry_bff1ad02-8ff9-43b4-8bc5-ef2b8baa5c8e"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:55.429519806Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_crucible_9129ddf3-00d8-4d5d-bf8e-e611448e5c48"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:55.4295259Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_nexus_9ff5ab27-a3ab-4401-8a23-c0799de85f2f"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:55.429531478Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_clickhouse_d5346055-3249-4357-951e-e5bf8d4dc3cf"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:55.429540328Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_crucible_a2c1f30f-a8a6-4e74-af9a-f32003a35978"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:55.429546384Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_crucible_pantry_be250b1e-18db-457d-9646-485f60896fb8"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:55.429552094Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_crucible_1f545bea-eb7a-44c2-a399-0237abf2a102"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:55.429558965Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_external_dns_544a0264-7c3a-4ada-aa6e-553800c85da2"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:55.429564451Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_external_dns_f0ddfaba-8652-4628-848a-f993e079fb94"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:55.42963516Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_crucible_e35814c6-9e18-4226-86a8-2ab44a0293b2"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:55.42964156Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_crucible_82bbc823-811c-4ea4-adbb-453cfe570acf"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:55.42964727Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_oximeter_da35a429-5291-41f8-8c2f-7377bb15ed38"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:55.42965337Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_crucible_41863173-79c0-4807-9ede-f5bf31ee3536"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:55.42966285Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_crucible_68e6a369-307f-4afa-a667-cfb114baa982"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:55.42966859Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2967","zone":"oxz_crucible_pantry_f8029ab0-8b4d-481d-8eec-fd6fa4c4e41c"} {"msg":"Writing ledger to /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:55.432013888Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:55.435765488Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:55.443566132Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"096fb2f1-7e38-417d-9f74-d0e5b7eab9b0","remote_addr":"[fd00:1122:3344:101::1]:35174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":42101239,"response_code":"204"} {"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:55.473570203Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1244"} {"msg":"Writing ledger to /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:55.473826015Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:55.473975668Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:55.480861887Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:681"} {"msg":"Nexus address: [fd00:1122:3344:101::c]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:55.490714755Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:694"} {"msg":"rack_network_config: RackNetworkConfigV2 {\n bfd: [],\n bgp: [],\n infra_ip_first: 10.151.2.174,\n infra_ip_last: 10.151.2.174,\n ports: [\n PortConfigV2 {\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.2.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n autoneg: false,\n bgp_peers: [],\n port: \"qsfp0\",\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.2.100,\n vlan_id: None,\n },\n ],\n switch: Switch0,\n uplink_port_fec: None,\n uplink_port_speed: Speed40G,\n },\n ],\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:55.551459164Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:825"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/444442ff-30f3-4f30-9be3-25f64879f808/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:55.559000845Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/444442ff-30f3-4f30-9be3-25f64879f808/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:55.622737512Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/444442ff-30f3-4f30-9be3-25f64879f808/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:55.728253715Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/444442ff-30f3-4f30-9be3-25f64879f808/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:55.884924395Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/444442ff-30f3-4f30-9be3-25f64879f808/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:56.143647669Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/444442ff-30f3-4f30-9be3-25f64879f808/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:56.313190695Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/444442ff-30f3-4f30-9be3-25f64879f808/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:56.490622997Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/444442ff-30f3-4f30-9be3-25f64879f808/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:56.879216826Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/444442ff-30f3-4f30-9be3-25f64879f808/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:57.097683329Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/444442ff-30f3-4f30-9be3-25f64879f808/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:57.578288631Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/444442ff-30f3-4f30-9be3-25f64879f808/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:58.016841192Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/444442ff-30f3-4f30-9be3-25f64879f808/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:58.608672765Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/444442ff-30f3-4f30-9be3-25f64879f808/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:07:59.555383003Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/444442ff-30f3-4f30-9be3-25f64879f808/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:00.168205367Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/444442ff-30f3-4f30-9be3-25f64879f808/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:01.605536073Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/444442ff-30f3-4f30-9be3-25f64879f808/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:02.645202364Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/444442ff-30f3-4f30-9be3-25f64879f808/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:03.79297677Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:04.774845789Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f17e4d64-ab29-4d93-816e-a9eeb2fc893d\", \"content-length\": \"173\", \"date\": \"Sat, 06 Jul 2024 00:08:05 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: rack with id \\\"444442ff-30f3-4f30-9be3-25f64879f808\\\"\", request_id: \"f17e4d64-ab29-4d93-816e-a9eeb2fc893d\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:05.985463899Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8c4d20c1-0c7c-4ea4-91eb-7b8785234334\", \"content-length\": \"173\", \"date\": \"Sat, 06 Jul 2024 00:08:08 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"4d88c6ce-c5a2-418b-b915-09fea24f811c\\\"\", request_id: \"8c4d20c1-0c7c-4ea4-91eb-7b8785234334\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:08.86013062Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4c12145f-3ab8-4c7f-ba73-d93dd54d14c6\", \"content-length\": \"173\", \"date\": \"Sat, 06 Jul 2024 00:08:11 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"4d88c6ce-c5a2-418b-b915-09fea24f811c\\\"\", request_id: \"4c12145f-3ab8-4c7f-ba73-d93dd54d14c6\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:11.896449552Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7308352c-0f38-480c-a912-5967d127246f\", \"content-length\": \"173\", \"date\": \"Sat, 06 Jul 2024 00:08:15 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"4d88c6ce-c5a2-418b-b915-09fea24f811c\\\"\", request_id: \"7308352c-0f38-480c-a912-5967d127246f\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:15.518149781Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"66c99b71-3ec8-40be-b285-8cb4a8f17c4f\", \"content-length\": \"173\", \"date\": \"Sat, 06 Jul 2024 00:08:19 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"4d88c6ce-c5a2-418b-b915-09fea24f811c\\\"\", request_id: \"66c99b71-3ec8-40be-b285-8cb4a8f17c4f\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:19.901481572Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:23.982901302Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:54609"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:23.982959283Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"adb1c050-fa58-489c-b34f-d55f20f83d4b","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:54609","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":39,"response_code":"200"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f760c7a4-c714-4027-9e62-39584603e856\", \"content-length\": \"173\", \"date\": \"Sat, 06 Jul 2024 00:08:25 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"4d88c6ce-c5a2-418b-b915-09fea24f811c\\\"\", request_id: \"f760c7a4-c714-4027-9e62-39584603e856\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:25.616059099Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f68f5488-e774-40c7-ab53-f5db80c82308\", \"content-length\": \"173\", \"date\": \"Sat, 06 Jul 2024 00:08:29 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"4d88c6ce-c5a2-418b-b915-09fea24f811c\\\"\", request_id: \"f68f5488-e774-40c7-ab53-f5db80c82308\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:29.870298148Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fe973243-c781-4431-aaa6-96f5f6da8323\", \"content-length\": \"173\", \"date\": \"Sat, 06 Jul 2024 00:08:37 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"4d88c6ce-c5a2-418b-b915-09fea24f811c\\\"\", request_id: \"fe973243-c781-4431-aaa6-96f5f6da8323\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:37.562688595Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9b11aa2a-1405-48a5-a743-139f8765f0c7\", \"content-length\": \"173\", \"date\": \"Sat, 06 Jul 2024 00:08:46 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"4d88c6ce-c5a2-418b-b915-09fea24f811c\\\"\", request_id: \"9b11aa2a-1405-48a5-a743-139f8765f0c7\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:46.35781775Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:46.6465342Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"sled-agent/src/nexus.rs:592","new_generation":"1","old_generation":"1"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:46.781403324Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:40656"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:46.783016049Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:46.873300811Z","hostname":"sock","pid":1555,"uri":"/inventory","method":"GET","req_id":"7ec60192-da7c-472f-997d-e283553e6bf3","remote_addr":"[fd00:1122:3344:101::b]:40656","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":91718,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:46.874592622Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:46.87513419Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:46.878988972Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:46.87949841Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"GET","req_id":"b1a53740-a5a9-45f7-8a70-fc36d652509b","remote_addr":"[fd00:1122:3344:101::b]:40656","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":5695,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:51.280783648Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:56314","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","component":"producer-server","component":"MetricsManager","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::d]:58565"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:51.292409233Z","hostname":"sock","pid":1555,"uri":"/4d88c6ce-c5a2-418b-b915-09fea24f811c","method":"GET","req_id":"05e5a256-f003-4848-94eb-3ad02145fe92","remote_addr":"[fd00:1122:3344:101::d]:58565","local_addr":"[fd00:1122:3344:101::1]:56314","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":10861,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:54.014208475Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:57555"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:54.014331276Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"f36daa06-448a-4ed2-a186-082561abc3df","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:57555","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":81,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:57.044761544Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:51306"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:57.047115872Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:725","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0ab9fb64-2e84-44e1-91ff-d472c798d3d1, kind: Service { id: f0ddfaba-8652-4628-848a-f993e079fb94 }, name: Name(\"external-dns-f0ddfaba-8652-4628-848a-f993e079fb94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 4d9df03f-9907-4449-8d22-3a38f32b899c, kind: Service { id: 544a0264-7c3a-4ada-aa6e-553800c85da2 }, name: Name(\"external-dns-544a0264-7c3a-4ada-aa6e-553800c85da2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 114])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 4ad4ab3a-b527-470c-b61b-a8987f3919b5, kind: Service { id: 8e46afa6-9da3-407c-92bf-128b95063f04 }, name: Name(\"nexus-8e46afa6-9da3-407c-92bf-128b95063f04\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 96bac47d-0e2f-4191-820f-fdc11f7f9342, kind: Service { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f }, name: Name(\"nexus-9ff5ab27-a3ab-4401-8a23-c0799de85f2f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ab5e3564-209c-48fa-b8f5-aa370048ac8d, kind: Service { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67 }, name: Name(\"nexus-7b52f14b-12cb-45e4-b4e3-5c517e3cba67\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 39])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:57.047296305Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:57.047332965Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:57.047342256Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:57.047351056Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:57.047380016Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:57.047396126Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:57.047425937Z","hostname":"sock","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"5b211fa8-ef29-4a55-a7f2-b07dcd2aa929","remote_addr":"[fd00:1122:3344:101::c]:51306","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":2378,"response_code":"204"} {"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:57.049282707Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:897"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:57.355203881Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:57197"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:57.355317713Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:57.449911844Z","hostname":"sock","pid":1555,"uri":"/inventory","method":"GET","req_id":"04509269-a32c-4145-bb52-a05b35e12ad3","remote_addr":"[fd00:1122:3344:101::c]:57197","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":94623,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:57.451047692Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Reading ledger from /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:57.451104243Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:57.45154825Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:57.451830484Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"GET","req_id":"77761c50-af96-49e8-b51d-6acddb783295","remote_addr":"[fd00:1122:3344:101::c]:57197","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":1545,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:57.49284533Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:48926"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:57.493107654Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:725","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0ab9fb64-2e84-44e1-91ff-d472c798d3d1, kind: Service { id: f0ddfaba-8652-4628-848a-f993e079fb94 }, name: Name(\"external-dns-f0ddfaba-8652-4628-848a-f993e079fb94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 4d9df03f-9907-4449-8d22-3a38f32b899c, kind: Service { id: 544a0264-7c3a-4ada-aa6e-553800c85da2 }, name: Name(\"external-dns-544a0264-7c3a-4ada-aa6e-553800c85da2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 114])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 4ad4ab3a-b527-470c-b61b-a8987f3919b5, kind: Service { id: 8e46afa6-9da3-407c-92bf-128b95063f04 }, name: Name(\"nexus-8e46afa6-9da3-407c-92bf-128b95063f04\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 96bac47d-0e2f-4191-820f-fdc11f7f9342, kind: Service { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f }, name: Name(\"nexus-9ff5ab27-a3ab-4401-8a23-c0799de85f2f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ab5e3564-209c-48fa-b8f5-aa370048ac8d, kind: Service { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67 }, name: Name(\"nexus-7b52f14b-12cb-45e4-b4e3-5c517e3cba67\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 39])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:57.493169335Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:57.493217325Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:57.493231236Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:57.493242506Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:57.493254666Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:57.493281446Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:57.493302767Z","hostname":"sock","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"4b76ec87-581f-47eb-a58b-190b8131fae3","remote_addr":"[fd00:1122:3344:101::b]:48926","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":372,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:57.493630312Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:60020"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:57.493808375Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:725","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0ab9fb64-2e84-44e1-91ff-d472c798d3d1, kind: Service { id: f0ddfaba-8652-4628-848a-f993e079fb94 }, name: Name(\"external-dns-f0ddfaba-8652-4628-848a-f993e079fb94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 4d9df03f-9907-4449-8d22-3a38f32b899c, kind: Service { id: 544a0264-7c3a-4ada-aa6e-553800c85da2 }, name: Name(\"external-dns-544a0264-7c3a-4ada-aa6e-553800c85da2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 114])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 4ad4ab3a-b527-470c-b61b-a8987f3919b5, kind: Service { id: 8e46afa6-9da3-407c-92bf-128b95063f04 }, name: Name(\"nexus-8e46afa6-9da3-407c-92bf-128b95063f04\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 96bac47d-0e2f-4191-820f-fdc11f7f9342, kind: Service { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f }, name: Name(\"nexus-9ff5ab27-a3ab-4401-8a23-c0799de85f2f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ab5e3564-209c-48fa-b8f5-aa370048ac8d, kind: Service { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67 }, name: Name(\"nexus-7b52f14b-12cb-45e4-b4e3-5c517e3cba67\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 39])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:57.493830285Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:57.493853555Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:57.493871266Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:57.493886656Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:57.493898676Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:57.493904096Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:57.493909056Z","hostname":"sock","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"58252b2b-7aec-4a0f-9cdf-3a1e022a235a","remote_addr":"[fd00:1122:3344:101::c]:60020","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":187,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:57.61437783Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:38475"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:57.614589643Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:725","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0ab9fb64-2e84-44e1-91ff-d472c798d3d1, kind: Service { id: f0ddfaba-8652-4628-848a-f993e079fb94 }, name: Name(\"external-dns-f0ddfaba-8652-4628-848a-f993e079fb94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 4d9df03f-9907-4449-8d22-3a38f32b899c, kind: Service { id: 544a0264-7c3a-4ada-aa6e-553800c85da2 }, name: Name(\"external-dns-544a0264-7c3a-4ada-aa6e-553800c85da2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 114])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 4ad4ab3a-b527-470c-b61b-a8987f3919b5, kind: Service { id: 8e46afa6-9da3-407c-92bf-128b95063f04 }, name: Name(\"nexus-8e46afa6-9da3-407c-92bf-128b95063f04\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 96bac47d-0e2f-4191-820f-fdc11f7f9342, kind: Service { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f }, name: Name(\"nexus-9ff5ab27-a3ab-4401-8a23-c0799de85f2f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ab5e3564-209c-48fa-b8f5-aa370048ac8d, kind: Service { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67 }, name: Name(\"nexus-7b52f14b-12cb-45e4-b4e3-5c517e3cba67\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 39])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:57.614647654Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:57.614680885Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:57.614697595Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:57.614703525Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:57.614725656Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:57.614745576Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:08:57.614769946Z","hostname":"sock","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"b5e00dbc-c00c-47df-87fe-d1831ef8239a","remote_addr":"[fd00:1122:3344:101::a]:38475","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":317,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:00.863855397Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:61879"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:00.864698681Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"0bf341d3-0da9-4d1d-ae8f-0532c244475c","remote_addr":"[fd00:1122:3344:101::c]:61879","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":821,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:00.936463227Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"0083f871-4758-4ceb-b899-0371b0891dfb","remote_addr":"[fd00:1122:3344:101::c]:61879","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":3766,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:06.116576741Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:50688"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:06.117478446Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"56258c32-cf34-4e49-aec4-71a5cd83e2b3","remote_addr":"[fd00:1122:3344:101::c]:50688","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":877,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:06.126806865Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"ae38e5d9-40c7-408d-ba25-a3eda4ba41aa","remote_addr":"[fd00:1122:3344:101::c]:50688","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":127,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:06.140089327Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:49387"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:06.140994802Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"50b02bd3-2780-4343-b854-a2f0798d8388","remote_addr":"[fd00:1122:3344:101::c]:49387","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":881,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:06.520969809Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:52151"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:06.651089556Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"76eeb166-33c4-485d-bac6-de56cb86a4b1","remote_addr":"[fd00:1122:3344:101::c]:52151","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":130043,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:06.651297219Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:60357"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:06.651413281Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:34872"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:06.651637745Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"8315da6e-bec7-4e50-a44e-bbf521f53207","remote_addr":"[fd00:1122:3344:101::b]:60357","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":276,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:06.653355032Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"fde8c5fc-dea4-458d-a372-846ead9e9c61","remote_addr":"[fd00:1122:3344:101::b]:34872","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":1819,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:06.654586202Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"1b03acfc-5130-4846-b90b-09f0aab6cbc0","remote_addr":"[fd00:1122:3344:101::c]:52151","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":3087,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:06.664301257Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"5e04b0f7-0f14-4a6d-82ab-d088bb65c9be","remote_addr":"[fd00:1122:3344:101::b]:34872","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":138,"response_code":"204"} {"msg":"Attempting to update network config with generation: 2, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:06.703688706Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:06.703823668Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:06.70517392Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 2","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:06.705967412Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:06.705984193Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"PUT","req_id":"9d446af7-7d4f-492b-aa62-d81b8f74baec","remote_addr":"[fd00:1122:3344:101::c]:52151","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":2841,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:06.943081419Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:62726"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:06.943277672Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"8b2da650-f4c5-4fa7-a452-37408dd25ec2","remote_addr":"[fd00:1122:3344:101::a]:62726","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":167,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:06.948731279Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:34556"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:06.948821461Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"2c39e80e-721a-4e67-a2ad-154d91f72f2d","remote_addr":"[fd00:1122:3344:101::a]:34556","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":36,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:06.960394135Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"6eaa5e0c-cfe7-40e1-82d5-119efbf1da05","remote_addr":"[fd00:1122:3344:101::a]:34556","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":940,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:07.076394698Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:65042"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:07.12918151Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"b2a069a9-b365-4a13-a4f7-0cfd5cf50275","remote_addr":"[fd00:1122:3344:101::b]:65042","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":52719,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:07.129533366Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"0ae5b711-85d1-4668-9284-6ba0e374948b","remote_addr":"[fd00:1122:3344:101::b]:65042","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":107,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:07.464202419Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:36241"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:07.520494088Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"d27b673e-ee12-47d7-8310-4d7826eae619","remote_addr":"[fd00:1122:3344:101::a]:36241","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":56230,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:07.520928915Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"74551a55-0641-4aee-a984-cc46f373716b","remote_addr":"[fd00:1122:3344:101::a]:36241","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":117,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:21.280516346Z","hostname":"sock","pid":1555,"uri":"/4d88c6ce-c5a2-418b-b915-09fea24f811c","method":"GET","req_id":"f89e04f4-7acb-423d-94ef-76674180fdb1","remote_addr":"[fd00:1122:3344:101::d]:58565","local_addr":"[fd00:1122:3344:101::1]:56314","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":884,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:24.044385868Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:32877"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:24.045024598Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"7eb16771-9a6a-4773-b8c7-23fd13dc8dd3","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:32877","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":615,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:35.863222007Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:40154"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:35.863329059Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"c4487b66-fd42-4435-8ec5-b448787c725d","remote_addr":"[fd00:1122:3344:101::c]:40154","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":104,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:35.871279956Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"213367b6-3fb6-4cf1-89a8-68fb4cfb434c","remote_addr":"[fd00:1122:3344:101::c]:40154","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":775,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:35.871335157Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:60434"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:35.871474899Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"79effb2e-a460-4335-94a1-cee011849b86","remote_addr":"[fd00:1122:3344:101::c]:60434","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":114,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:36.132955214Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:56413"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:36.183743516Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"661612c0-39ee-48ef-b81c-9589e6fcdcb9","remote_addr":"[fd00:1122:3344:101::c]:56413","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":50733,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:36.184267204Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"6924b784-5739-4bff-a879-e3377f87f89d","remote_addr":"[fd00:1122:3344:101::c]:56413","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":103,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:36.500429841Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:64767"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:36.500582574Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"b17b6890-f820-412d-9a52-489cf63e44eb","remote_addr":"[fd00:1122:3344:101::b]:64767","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":98,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:36.509971304Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"f111465e-3432-4b9e-b53c-ba7941f9d2e0","remote_addr":"[fd00:1122:3344:101::b]:64767","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":144,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:36.611455285Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:56478"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:36.611642157Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"c3a01df6-c9e2-45cf-ad1d-db9d0082eb62","remote_addr":"[fd00:1122:3344:101::b]:56478","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":153,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:36.644559573Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:55193"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:36.645208133Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"2dee7a90-fa21-4198-831b-9f5c51bf482a","remote_addr":"[fd00:1122:3344:101::a]:55193","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":623,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:36.652462189Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"ced3019e-dfea-48f4-9f8a-11b13b311636","remote_addr":"[fd00:1122:3344:101::a]:55193","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":105,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:36.675386735Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:40424"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:36.729897866Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"a041fdd6-210c-4360-8970-86d26ea0c49b","remote_addr":"[fd00:1122:3344:101::b]:40424","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":54474,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:36.73013999Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:50248"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:36.730240411Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"b6c120c3-62d6-457b-9f03-15d9a0ce8f46","remote_addr":"[fd00:1122:3344:101::b]:40424","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":134,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:36.73139369Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"ba9ac4e6-8c1a-4637-99e0-348ed7419ec8","remote_addr":"[fd00:1122:3344:101::a]:50248","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":1210,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:36.819859182Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:64694"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:36.868247665Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"2e5a93f8-e6fc-490e-91fd-3ee56165bf77","remote_addr":"[fd00:1122:3344:101::a]:64694","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":48308,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:36.868523989Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"76669b32-0193-4285-99e9-26f8572be912","remote_addr":"[fd00:1122:3344:101::a]:64694","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":106,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:51.279649082Z","hostname":"sock","pid":1555,"uri":"/4d88c6ce-c5a2-418b-b915-09fea24f811c","method":"GET","req_id":"6f72861d-3d59-497f-92f3-1987cb0c0025","remote_addr":"[fd00:1122:3344:101::d]:58565","local_addr":"[fd00:1122:3344:101::1]:56314","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":1081,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:54.076912607Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45775"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:09:54.077005119Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"49ade7c8-6bad-40b7-b1ac-866e133a668a","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45775","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":86,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:06.048779087Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:57231"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:06.04899488Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"127f8e77-a82a-40ad-ac11-35ec47f41f9a","remote_addr":"[fd00:1122:3344:101::c]:57231","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":200,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:06.101445228Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:34295"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:06.101538159Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"c50599d5-29fc-49bb-8e2d-0dbc76fd6367","remote_addr":"[fd00:1122:3344:101::c]:34295","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":71,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:06.11094494Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"b4c5a447-db5b-461c-8be5-0a06c48f5485","remote_addr":"[fd00:1122:3344:101::c]:34295","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":753,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:06.280450216Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:64712"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:06.325984593Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"c318d8f5-fe8c-4463-a770-2067fe1fee92","remote_addr":"[fd00:1122:3344:101::c]:64712","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":45493,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:06.326275337Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"b57acbb7-7b68-48cf-92c2-03873cc72fb5","remote_addr":"[fd00:1122:3344:101::c]:64712","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":104,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:06.731739412Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:43621"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:06.731884414Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"a6e02b48-173a-4457-b75c-37d8deb6953f","remote_addr":"[fd00:1122:3344:101::a]:43621","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":117,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:06.739728379Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"db4894ca-6167-4925-96ff-4dec704e80f9","remote_addr":"[fd00:1122:3344:101::a]:43621","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":133,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:06.780869676Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:44499"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:06.781049219Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"b09c3358-fadb-4d03-9fae-3d06fbdaac48","remote_addr":"[fd00:1122:3344:101::a]:44499","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":157,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:06.808848263Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:38935"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:06.808932784Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"09a83619-996c-4be3-9c3c-03235e3e483c","remote_addr":"[fd00:1122:3344:101::b]:38935","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":37,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:06.818340544Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"b7442ed7-e1c7-42d0-8a3d-94ee729b158d","remote_addr":"[fd00:1122:3344:101::b]:38935","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":833,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:06.852456429Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:36805"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:06.852587891Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"781aa8cd-d505-4954-b5d7-4ba49765dd88","remote_addr":"[fd00:1122:3344:101::b]:36805","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":89,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:06.968295269Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:46719"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:07.016281495Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"28acd772-8cd2-4d66-8e5e-bc162d9d4e27","remote_addr":"[fd00:1122:3344:101::a]:46719","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":47936,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:07.016479528Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:44957"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:07.020072876Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"8f63f871-ba41-4822-bac7-70e14932beaa","remote_addr":"[fd00:1122:3344:101::a]:46719","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":3461,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:07.061117831Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"65520b99-ab83-47b0-9da5-21aea41a71ef","remote_addr":"[fd00:1122:3344:101::b]:44957","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":44560,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:07.061619409Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"32397789-e5e0-4d30-be5c-b11d76cd3c31","remote_addr":"[fd00:1122:3344:101::b]:44957","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":240,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:21.27862263Z","hostname":"sock","pid":1555,"uri":"/4d88c6ce-c5a2-418b-b915-09fea24f811c","method":"GET","req_id":"3d239896-9f88-416b-b61d-d466d956946f","remote_addr":"[fd00:1122:3344:101::d]:58565","local_addr":"[fd00:1122:3344:101::1]:56314","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":621,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:24.108734489Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:36634"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:24.108862791Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"c02d1058-42a1-41ae-a7d9-3328fd54bd4e","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:36634","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":93,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:35.878692615Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:57394"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:35.878830227Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"07cb105c-53ae-4c97-ba8b-4c2d958f3f16","remote_addr":"[fd00:1122:3344:101::c]:57394","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":111,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:35.887004928Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"b99931bc-d73c-4c7a-a3a3-3ba86fffebdf","remote_addr":"[fd00:1122:3344:101::c]:57394","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":895,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:35.991486746Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:52664"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:35.992222768Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"9a9fb024-ae58-43ee-969e-f6427cb13a23","remote_addr":"[fd00:1122:3344:101::c]:52664","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":722,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:36.091233019Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:55350"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:36.140521066Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"c6089b53-e29e-4535-8cd7-8a4df727d57e","remote_addr":"[fd00:1122:3344:101::c]:55350","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":49216,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:36.140828511Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"1f96dae7-1024-4656-961f-8a800e243964","remote_addr":"[fd00:1122:3344:101::c]:55350","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":114,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:36.567113167Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:40057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:36.56726303Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"4e395652-9f2c-4a23-bd14-8732ca730898","remote_addr":"[fd00:1122:3344:101::a]:40057","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":126,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:36.574935162Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"9156dd14-5a7a-4187-a47e-3ed315e7f736","remote_addr":"[fd00:1122:3344:101::a]:40057","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":148,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:36.616583227Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:50272"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:36.617867948Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"e862e02a-dab0-4241-a90c-fe647cd9455f","remote_addr":"[fd00:1122:3344:101::b]:50272","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":1228,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:36.667695193Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:46945"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:36.668568967Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"88ad4ecc-0908-4e60-97ad-bbf0640b48e0","remote_addr":"[fd00:1122:3344:101::b]:46945","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":852,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:36.676791139Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"02a0905d-c05f-4565-a42b-3e74867ba738","remote_addr":"[fd00:1122:3344:101::b]:46945","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":65,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:36.72697741Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:57726"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:36.727123752Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"66e49090-6270-4109-9475-eef73ea507fe","remote_addr":"[fd00:1122:3344:101::a]:57726","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":93,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:36.800759218Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:41898"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:36.847997652Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"955d1422-22ce-4af4-b47e-cf638a3984b1","remote_addr":"[fd00:1122:3344:101::a]:41898","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":47170,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:36.848130715Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:56432"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:36.851369446Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"e7e7f9bc-b992-469b-8d5b-d2c9a3bb73a6","remote_addr":"[fd00:1122:3344:101::a]:41898","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":2947,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:36.893276905Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"b6049c36-2b68-454d-a21c-0b7f61d9a472","remote_addr":"[fd00:1122:3344:101::b]:56432","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":45035,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:36.893668212Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"4d464577-510d-475e-bd3b-2ae6dbb8e5b3","remote_addr":"[fd00:1122:3344:101::b]:56432","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":152,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:51.288183906Z","hostname":"sock","pid":1555,"uri":"/4d88c6ce-c5a2-418b-b915-09fea24f811c","method":"GET","req_id":"bbe7e15b-fc6e-4d82-9d31-d17fa351367b","remote_addr":"[fd00:1122:3344:101::d]:58565","local_addr":"[fd00:1122:3344:101::1]:56314","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":278,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:54.140732753Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:64160"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:10:54.140789754Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"2ce7d6fd-6a1c-4c1e-a8bf-0df46367210b","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:64160","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":30,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:05.97966597Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:55505"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:05.979812702Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"f670657b-b6c5-49b4-b9bc-b41592a75cb4","remote_addr":"[fd00:1122:3344:101::c]:55505","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":127,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:05.989850953Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"12495f13-5f4d-4c24-a1ed-a4b089e54556","remote_addr":"[fd00:1122:3344:101::c]:55505","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":814,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:06.175321294Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:39018"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:06.175501807Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"c4f6e7f5-067f-46e2-941e-d7ace66ae470","remote_addr":"[fd00:1122:3344:101::c]:39018","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":150,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:06.295724267Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:63130"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:06.337070266Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"1bdd9d25-4400-442d-beab-762fb88e438d","remote_addr":"[fd00:1122:3344:101::c]:63130","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":41281,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:06.33733366Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"1813a8c9-988e-4968-b65c-871f02930575","remote_addr":"[fd00:1122:3344:101::c]:63130","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":102,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:06.689382032Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:65170"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:06.689524284Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"d150c4e5-b0a6-494b-a548-87fec1a3afe5","remote_addr":"[fd00:1122:3344:101::a]:65170","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":98,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:06.698289104Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"61b7b664-3d4e-46eb-bc51-ccc5f5ab6c87","remote_addr":"[fd00:1122:3344:101::a]:65170","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":135,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:06.751744487Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:44332"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:06.751854949Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"41b78b64-b423-4edc-ac7e-7315355f6fc3","remote_addr":"[fd00:1122:3344:101::b]:44332","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":33,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:06.758741489Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"4be26c83-a332-495f-a740-e4bb1582fa49","remote_addr":"[fd00:1122:3344:101::b]:44332","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":66,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:06.799881556Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:53557"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:06.800904192Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"b8216196-5f08-4004-9df2-2cc111ea0653","remote_addr":"[fd00:1122:3344:101::b]:53557","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":961,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:06.858884128Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:39877"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:06.85902273Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"85762cef-17be-4b6d-bb45-944f610c3702","remote_addr":"[fd00:1122:3344:101::a]:39877","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":114,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:06.909993394Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:62226"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:06.959419394Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"8d86534c-fdfd-452d-91ec-28928096c4f6","remote_addr":"[fd00:1122:3344:101::b]:62226","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":49367,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:06.959695818Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"a7cd7397-dff8-40a7-8a91-8de273457725","remote_addr":"[fd00:1122:3344:101::b]:62226","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":109,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:06.973426527Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:47816"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:07.018026009Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"6d8782c9-13e2-4ac5-aacc-ee817219efaf","remote_addr":"[fd00:1122:3344:101::a]:47816","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":44517,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:07.018296204Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"2057cdf8-52c6-49dd-99d4-69f210d38f9f","remote_addr":"[fd00:1122:3344:101::a]:47816","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":97,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:21.327733942Z","hostname":"sock","pid":1555,"uri":"/4d88c6ce-c5a2-418b-b915-09fea24f811c","method":"GET","req_id":"3cbc2b8f-719f-41e2-ba0f-0fffc593578b","remote_addr":"[fd00:1122:3344:101::d]:58565","local_addr":"[fd00:1122:3344:101::1]:56314","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":1199,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:24.173199277Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:37351"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:24.173251198Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"6f7356bf-00a9-4411-b4c9-23d5a752c911","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:37351","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":34,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:35.859409622Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:46830"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:35.859560974Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"f26b3c34-d1bf-457a-ab1c-5966a001dc08","remote_addr":"[fd00:1122:3344:101::c]:46830","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":131,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:35.866350702Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"36b15e63-700d-4576-aaf4-14d4dd330ca2","remote_addr":"[fd00:1122:3344:101::c]:46830","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":124,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:35.882271327Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:61164"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:35.882412549Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"f5f912f1-be7b-443a-867f-744e7bbbae4c","remote_addr":"[fd00:1122:3344:101::c]:61164","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":120,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:36.025650386Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:60344"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:36.07411584Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"a66d3bd7-ac4d-48dc-ae58-eea7314760d7","remote_addr":"[fd00:1122:3344:101::c]:60344","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":48396,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:36.074376704Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"a5d3d1fb-4987-468a-8971-609932a54739","remote_addr":"[fd00:1122:3344:101::c]:60344","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":109,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:36.545865763Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:40990"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:36.546272749Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"49deb9eb-8d97-449a-97de-dc0dd2a8ec40","remote_addr":"[fd00:1122:3344:101::b]:40990","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":387,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:36.553899441Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"908bf285-ff59-45c9-b284-8d4bac5d062a","remote_addr":"[fd00:1122:3344:101::b]:40990","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":119,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:36.555993784Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:58965"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:36.556149577Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"9070ab9c-e5b1-4c05-825c-341b25c1a31f","remote_addr":"[fd00:1122:3344:101::b]:58965","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":107,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:36.619744852Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:43925"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:36.619837194Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"0448729f-a898-4e19-9e8a-31d3e66ac0ec","remote_addr":"[fd00:1122:3344:101::a]:43925","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":40,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:36.626906097Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"483d21ee-21ff-4446-a9cd-c0e2ef18c510","remote_addr":"[fd00:1122:3344:101::a]:43925","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":67,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:36.62713627Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:41738"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:36.627898732Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"c77477d3-2b91-4160-a744-6bc9a4221170","remote_addr":"[fd00:1122:3344:101::a]:41738","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":732,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:36.766384834Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:64640"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:36.814041105Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"f0af1c2f-1d29-4619-8030-887b9c2f36c7","remote_addr":"[fd00:1122:3344:101::b]:64640","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":47617,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:36.814158247Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:50778"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:36.817715143Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"254cc7c6-9507-4ed1-813d-c4f6ef75b0ea","remote_addr":"[fd00:1122:3344:101::b]:64640","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":3316,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:36.85756748Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"fe5ec3e4-d4e3-495c-b52d-0580cc48ced9","remote_addr":"[fd00:1122:3344:101::a]:50778","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":43317,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:36.857918445Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"97b78f35-5fd9-44a8-a3d0-d660ad94a74a","remote_addr":"[fd00:1122:3344:101::a]:50778","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":123,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:51.326961645Z","hostname":"sock","pid":1555,"uri":"/4d88c6ce-c5a2-418b-b915-09fea24f811c","method":"GET","req_id":"82f9d1fa-f912-41b1-944d-2050e81a6257","remote_addr":"[fd00:1122:3344:101::d]:58565","local_addr":"[fd00:1122:3344:101::1]:56314","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":525,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:54.204839027Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:53590"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:11:54.204908978Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"1aa9cc7f-dc50-4000-9b65-defa973ee2f1","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:53590","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":49,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:05.953970625Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:63948"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:05.954085147Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"2c0a8d35-6b0f-4116-8cc8-52b6af32c97f","remote_addr":"[fd00:1122:3344:101::c]:63948","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":109,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:05.954764988Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:55894"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:05.95493523Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"e6363dda-5242-4f50-bdb4-96407a0c9191","remote_addr":"[fd00:1122:3344:101::c]:55894","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":142,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:05.962142716Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"3c4b09b8-59f1-47ee-a107-ffe84d31f9a9","remote_addr":"[fd00:1122:3344:101::c]:63948","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":104,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:06.168239726Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:51156"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:06.21791524Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"fee1f065-5aba-43f0-a256-d629f88abf9d","remote_addr":"[fd00:1122:3344:101::c]:51156","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":49592,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:06.218219954Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"eb0c2f04-9dcb-4f26-a4a9-150323abba44","remote_addr":"[fd00:1122:3344:101::c]:51156","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":93,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:06.599913639Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:54618"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:06.600132113Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"0e67723e-0023-486f-9f60-597dc6ac0917","remote_addr":"[fd00:1122:3344:101::b]:54618","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":179,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:06.627282716Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:59727"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:06.628320163Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"b3a7809b-cf2c-4054-9344-f0957bf8580c","remote_addr":"[fd00:1122:3344:101::b]:59727","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":1012,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:06.635907624Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"335aa477-6d1d-4b0b-bba8-86a8b26e6326","remote_addr":"[fd00:1122:3344:101::b]:59727","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":113,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:06.717261913Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:44584"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:06.717355484Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"f92e1718-f9c6-40cb-ac80-eae5f53b181b","remote_addr":"[fd00:1122:3344:101::a]:44584","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":35,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:06.725180109Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"fc749110-5e66-4606-87e7-7ea99f30fb48","remote_addr":"[fd00:1122:3344:101::a]:44584","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":80,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:06.898377025Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:63968"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:06.898517667Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"5a50bbba-0b43-40df-b310-8ce1c63bd939","remote_addr":"[fd00:1122:3344:101::a]:63968","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":115,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:06.96758669Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:48533"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:07.013895299Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"b69a02e2-459b-4d0e-a5e5-eb6695e1417e","remote_addr":"[fd00:1122:3344:101::b]:48533","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":46239,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:07.014210835Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"8e6afc7f-bfd5-42ca-bdc6-b8b359053bf2","remote_addr":"[fd00:1122:3344:101::b]:48533","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":116,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:07.109052389Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:43819"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:07.154561405Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"ffba9ec2-865a-40fb-910a-2eecad1f241f","remote_addr":"[fd00:1122:3344:101::a]:43819","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":45444,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:07.155560681Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"fdf6e0db-f84d-4f75-9a48-2db9c5a14a9a","remote_addr":"[fd00:1122:3344:101::a]:43819","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":802,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:21.331202133Z","hostname":"sock","pid":1555,"uri":"/4d88c6ce-c5a2-418b-b915-09fea24f811c","method":"GET","req_id":"c42d2b1e-1fe4-4bf4-b75f-65570ba40fd8","remote_addr":"[fd00:1122:3344:101::d]:58565","local_addr":"[fd00:1122:3344:101::1]:56314","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":1131,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:24.236516443Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:61072"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:24.236579944Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"efb63d7c-2955-4ede-b3ab-56be71878457","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:61072","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":40,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:35.847213193Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:34860"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:35.847838632Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"6b7c221a-f9a2-4994-8de3-6fb937a5b5af","remote_addr":"[fd00:1122:3344:101::c]:34860","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":609,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:35.8552033Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"5ceabb99-b4d7-4b04-9b9b-30893f0d88cb","remote_addr":"[fd00:1122:3344:101::c]:34860","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":124,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:35.861316628Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:61108"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:35.8614853Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"4a38f5de-88fb-41d2-be19-e73d209c2746","remote_addr":"[fd00:1122:3344:101::c]:61108","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":109,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:35.990828266Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:55725"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:36.042820096Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"d72500ea-759e-445e-a166-0cb116b61bd6","remote_addr":"[fd00:1122:3344:101::c]:55725","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":51928,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:36.043156451Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"bcf123ba-fa56-44f0-9d13-51ed127504fe","remote_addr":"[fd00:1122:3344:101::c]:55725","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":110,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:36.535152226Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:47029"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:36.535308459Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"4dcdf69a-50d8-4f35-b846-b3e97bf31a44","remote_addr":"[fd00:1122:3344:101::b]:47029","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":98,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:36.542513424Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"87b068a9-86cd-4fe8-9174-f38d1df11cbd","remote_addr":"[fd00:1122:3344:101::b]:47029","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":120,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:36.576493786Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:48980"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:36.577482412Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"679eca99-3902-4507-ad2d-329c4205cdc2","remote_addr":"[fd00:1122:3344:101::b]:48980","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":963,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:36.618771572Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:33564"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:36.618862913Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"04c0cf36-a4a9-43e6-b629-c665b32eaf7c","remote_addr":"[fd00:1122:3344:101::a]:33564","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":36,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:36.625466288Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"bf392d95-b1a8-4bfb-8724-0ec465217557","remote_addr":"[fd00:1122:3344:101::a]:33564","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":75,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:36.627919918Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:62466"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:36.628807882Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"4f98f621-e69f-4e41-b482-6bd971b119cb","remote_addr":"[fd00:1122:3344:101::a]:62466","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":864,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:36.708978022Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:56181"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:36.757512657Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"b43bf638-1651-467f-92e8-639cddfa04eb","remote_addr":"[fd00:1122:3344:101::b]:56181","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":48486,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:36.757634599Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:64251"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:36.761692354Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"60ca524d-da76-40bf-ae77-f28f130cf32f","remote_addr":"[fd00:1122:3344:101::b]:56181","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":3832,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:36.802544866Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"aa6a312a-9682-4136-9708-778478cd9565","remote_addr":"[fd00:1122:3344:101::a]:64251","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":44804,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:36.802912552Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"40f537c0-c610-4520-a1a5-f3ed7c87d390","remote_addr":"[fd00:1122:3344:101::a]:64251","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":137,"response_code":"200"} {"msg":"running automatic periodic zone bundle cleanup","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:51.234496436Z","hostname":"sock","pid":1555,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:300"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:51.234585238Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:51.278367137Z","hostname":"sock","pid":1555,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:289","time_to_next_cleanup":"599.99999582s","next_cleanup":"Instant { tv_sec: 2749, tv_nsec: 912259718 }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:51.327035963Z","hostname":"sock","pid":1555,"uri":"/4d88c6ce-c5a2-418b-b915-09fea24f811c","method":"GET","req_id":"bacbda99-8420-4b4d-9e34-ad4c8b45ea3d","remote_addr":"[fd00:1122:3344:101::d]:58565","local_addr":"[fd00:1122:3344:101::1]:56314","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":596,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:54.27034881Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:35382"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:12:54.271115273Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"a34b10f2-0ae3-49a9-aeb3-c5ff35e578f8","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:35382","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":740,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:04.774057134Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:06.085990391Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:46277"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:06.086247815Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:725","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0ab9fb64-2e84-44e1-91ff-d472c798d3d1, kind: Service { id: f0ddfaba-8652-4628-848a-f993e079fb94 }, name: Name(\"external-dns-f0ddfaba-8652-4628-848a-f993e079fb94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 4d9df03f-9907-4449-8d22-3a38f32b899c, kind: Service { id: 544a0264-7c3a-4ada-aa6e-553800c85da2 }, name: Name(\"external-dns-544a0264-7c3a-4ada-aa6e-553800c85da2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 114])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 4ad4ab3a-b527-470c-b61b-a8987f3919b5, kind: Service { id: 8e46afa6-9da3-407c-92bf-128b95063f04 }, name: Name(\"nexus-8e46afa6-9da3-407c-92bf-128b95063f04\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 96bac47d-0e2f-4191-820f-fdc11f7f9342, kind: Service { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f }, name: Name(\"nexus-9ff5ab27-a3ab-4401-8a23-c0799de85f2f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ab5e3564-209c-48fa-b8f5-aa370048ac8d, kind: Service { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67 }, name: Name(\"nexus-7b52f14b-12cb-45e4-b4e3-5c517e3cba67\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 39])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:06.086302856Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:06.086343607Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:06.086353767Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:06.086361327Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:06.086375617Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:06.086403598Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:06.086429589Z","hostname":"sock","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"60f427b2-2858-4412-9f5a-2f94132cb60d","remote_addr":"[fd00:1122:3344:101::c]:46277","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":377,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:06.119863692Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:41200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:06.119958324Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"18c05698-2c12-4467-8dfa-5ce4f53aa49b","remote_addr":"[fd00:1122:3344:101::c]:41200","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":64,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:06.128018913Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"09c84bd8-7cfa-4807-a47d-f206886af0a0","remote_addr":"[fd00:1122:3344:101::c]:41200","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":939,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:06.169022267Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:64177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:06.169157359Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"8edf9b72-9c1b-4d4f-9f69-660def24773a","remote_addr":"[fd00:1122:3344:101::c]:64177","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":115,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:06.313825389Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:56908"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:06.359504699Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"a72b27e8-9dde-4bfc-991f-5fd47b593753","remote_addr":"[fd00:1122:3344:101::c]:56908","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":45628,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:06.359813844Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"fba3fd2c-26f5-439a-bdbe-96ee73ce4a7a","remote_addr":"[fd00:1122:3344:101::c]:56908","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":104,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:06.739542177Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:48504"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:06.740497762Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"5648a7df-9506-4712-881a-a963d2901aa7","remote_addr":"[fd00:1122:3344:101::a]:48504","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":900,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:06.741012471Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:58404"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:06.741200214Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"f1a629ec-d150-451f-97c0-4408e94e7c01","remote_addr":"[fd00:1122:3344:101::a]:58404","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":149,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:06.748227746Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"9b491c44-40f1-4287-b40a-5f7a53b6ad96","remote_addr":"[fd00:1122:3344:101::a]:48504","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":871,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:06.804164409Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:47953"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:06.805224156Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"fab7b9fd-7f16-4bdd-8216-ee5acdc63c65","remote_addr":"[fd00:1122:3344:101::b]:47953","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":1037,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:06.829224551Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:52732"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:06.829305524Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"71346e6d-54df-4d06-8556-5abcfb6798ec","remote_addr":"[fd00:1122:3344:101::b]:52732","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":37,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:06.837289378Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"7ead6ec8-e1e1-4b74-b290-9600619f5fd3","remote_addr":"[fd00:1122:3344:101::b]:52732","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":84,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:06.861048217Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:34289"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:06.861249281Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:725","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0ab9fb64-2e84-44e1-91ff-d472c798d3d1, kind: Service { id: f0ddfaba-8652-4628-848a-f993e079fb94 }, name: Name(\"external-dns-f0ddfaba-8652-4628-848a-f993e079fb94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 4d9df03f-9907-4449-8d22-3a38f32b899c, kind: Service { id: 544a0264-7c3a-4ada-aa6e-553800c85da2 }, name: Name(\"external-dns-544a0264-7c3a-4ada-aa6e-553800c85da2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 114])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 4ad4ab3a-b527-470c-b61b-a8987f3919b5, kind: Service { id: 8e46afa6-9da3-407c-92bf-128b95063f04 }, name: Name(\"nexus-8e46afa6-9da3-407c-92bf-128b95063f04\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 96bac47d-0e2f-4191-820f-fdc11f7f9342, kind: Service { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f }, name: Name(\"nexus-9ff5ab27-a3ab-4401-8a23-c0799de85f2f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ab5e3564-209c-48fa-b8f5-aa370048ac8d, kind: Service { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67 }, name: Name(\"nexus-7b52f14b-12cb-45e4-b4e3-5c517e3cba67\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 39])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:06.861279691Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:06.861320212Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:06.861336682Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:06.861349842Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:06.861367602Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:06.861385923Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:06.861401643Z","hostname":"sock","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"26545feb-2a17-4fbf-832a-39cf0bfa920a","remote_addr":"[fd00:1122:3344:101::a]:34289","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":324,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:06.875497598Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:46739"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:06.876031717Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:725","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0ab9fb64-2e84-44e1-91ff-d472c798d3d1, kind: Service { id: f0ddfaba-8652-4628-848a-f993e079fb94 }, name: Name(\"external-dns-f0ddfaba-8652-4628-848a-f993e079fb94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 4d9df03f-9907-4449-8d22-3a38f32b899c, kind: Service { id: 544a0264-7c3a-4ada-aa6e-553800c85da2 }, name: Name(\"external-dns-544a0264-7c3a-4ada-aa6e-553800c85da2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 114])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 4ad4ab3a-b527-470c-b61b-a8987f3919b5, kind: Service { id: 8e46afa6-9da3-407c-92bf-128b95063f04 }, name: Name(\"nexus-8e46afa6-9da3-407c-92bf-128b95063f04\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 96bac47d-0e2f-4191-820f-fdc11f7f9342, kind: Service { id: 9ff5ab27-a3ab-4401-8a23-c0799de85f2f }, name: Name(\"nexus-9ff5ab27-a3ab-4401-8a23-c0799de85f2f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ab5e3564-209c-48fa-b8f5-aa370048ac8d, kind: Service { id: 7b52f14b-12cb-45e4-b4e3-5c517e3cba67 }, name: Name(\"nexus-7b52f14b-12cb-45e4-b4e3-5c517e3cba67\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 39])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:06.876060107Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:06.876096167Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:06.876109028Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:06.876118398Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:06.876127818Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:06.876146418Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:744","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:06.876176189Z","hostname":"sock","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"8ef6aa2f-8567-4b85-a80a-bc0e0e261152","remote_addr":"[fd00:1122:3344:101::b]:46739","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":625,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:07.036890205Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:50947"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:07.087845141Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"e77e4083-5742-41a7-b06f-8fafa1b89cad","remote_addr":"[fd00:1122:3344:101::b]:50947","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":50963,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:07.088742473Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"9c59c533-f2ca-46ab-ba52-c64f82e10a90","remote_addr":"[fd00:1122:3344:101::b]:50947","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":669,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:07.088877025Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:50212"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:07.134228389Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"b198f985-3767-4a08-ad08-a342a4ffc063","remote_addr":"[fd00:1122:3344:101::a]:50212","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":45262,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:07.134524474Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"00decf4d-aa92-4bae-a54f-c484e44571c2","remote_addr":"[fd00:1122:3344:101::a]:50212","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":115,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:21.328845087Z","hostname":"sock","pid":1555,"uri":"/4d88c6ce-c5a2-418b-b915-09fea24f811c","method":"GET","req_id":"0825d9eb-7b7c-43ec-93c6-8d85a3e1119b","remote_addr":"[fd00:1122:3344:101::d]:58565","local_addr":"[fd00:1122:3344:101::1]:56314","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":1321,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:24.302723893Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:59424"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:24.302793194Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"9fa6468f-2711-41c9-9530-920ca98860db","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:59424","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":48,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:35.890283701Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:55008"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:35.890483055Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"34e4a77f-d8fc-4ba0-900d-e27abcd2550b","remote_addr":"[fd00:1122:3344:101::c]:55008","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":177,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:35.959462776Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:45603"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:35.959548437Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"f5702f48-8bf3-4ea4-820d-4b96ad16d597","remote_addr":"[fd00:1122:3344:101::c]:45603","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":66,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:35.966913145Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"9ae27fc3-02e8-4a17-96dd-703b906c732f","remote_addr":"[fd00:1122:3344:101::c]:45603","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":98,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:36.036465366Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:49682"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:36.087907447Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"da84525c-ba89-4821-8009-21ed72a58951","remote_addr":"[fd00:1122:3344:101::c]:49682","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":51399,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:36.088237912Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"8621cb0c-abea-43a0-8749-1c0a0af4e196","remote_addr":"[fd00:1122:3344:101::c]:49682","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":101,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:36.505770539Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:38049"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:36.505907031Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"c1dd6dfd-cf49-4c20-ae40-306f67052d6b","remote_addr":"[fd00:1122:3344:101::b]:38049","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":90,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:36.513098266Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"9e676ca1-a7da-4281-9dcc-d581f4b19e97","remote_addr":"[fd00:1122:3344:101::b]:38049","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":148,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:36.573086554Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:63810"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:36.573259516Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"b533f4a8-9de2-46ec-b1ca-12d22422a58e","remote_addr":"[fd00:1122:3344:101::b]:63810","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":140,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:36.656575997Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:58956"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:36.706309071Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"a8d4feea-753b-48a9-a46e-8ea605970e03","remote_addr":"[fd00:1122:3344:101::b]:58956","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":49682,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:36.706423563Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:48387"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:36.706639616Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"c93bd7a8-20e7-4f2e-8fa1-45a7b2e904f1","remote_addr":"[fd00:1122:3344:101::a]:48387","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":169,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:36.706768458Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"3aef9e84-4cf8-4d36-909c-5fbf3a4a3e60","remote_addr":"[fd00:1122:3344:101::b]:58956","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":203,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:36.717412928Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:64338"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:36.71754531Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"c7aaa49b-bd79-47ac-9234-423e60fc8bb2","remote_addr":"[fd00:1122:3344:101::a]:64338","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":81,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:36.72502587Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"5df3a76b-1492-48ef-9a38-ddaef81ed5b9","remote_addr":"[fd00:1122:3344:101::a]:64338","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":126,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:36.823073125Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:51009"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:36.873780825Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"63795771-5dfd-4343-873e-0d8b9c896595","remote_addr":"[fd00:1122:3344:101::a]:51009","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":50659,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:36.874178341Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"51475695-45f4-4db4-8b04-009d4732fe35","remote_addr":"[fd00:1122:3344:101::a]:51009","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":111,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:51.434446678Z","hostname":"sock","pid":1555,"uri":"/4d88c6ce-c5a2-418b-b915-09fea24f811c","method":"GET","req_id":"2a35d344-211c-47fe-bfc1-049f2882fbfd","remote_addr":"[fd00:1122:3344:101::d]:58565","local_addr":"[fd00:1122:3344:101::1]:56314","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":550,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:54.334021896Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:50677"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:13:54.334844149Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"6ec74a0f-92c5-4f07-b6c1-a4303c05d9bb","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:50677","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":788,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:06.072840033Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:40470"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:06.072993625Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"17147e91-e457-462d-b383-d1ebed490fba","remote_addr":"[fd00:1122:3344:101::c]:40470","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":118,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:06.089438158Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"814de3af-d173-47d8-8989-cc5f303c1be4","remote_addr":"[fd00:1122:3344:101::c]:40470","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":138,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:06.112833471Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:48402"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:06.113044795Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"119f22b7-17fa-4bb2-b9be-bcf5bc58c751","remote_addr":"[fd00:1122:3344:101::c]:48402","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":154,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:06.292210426Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:46942"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:06.342518072Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"8df5c3c2-56c4-4bed-b690-77004ea488d6","remote_addr":"[fd00:1122:3344:101::c]:46942","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":50303,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:06.342832593Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"6193d942-3f66-46b9-be22-43a84bcaad49","remote_addr":"[fd00:1122:3344:101::c]:46942","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":92,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:06.625926043Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:41474"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:06.626067736Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"835b73d1-c577-415c-99bb-575f4f08d370","remote_addr":"[fd00:1122:3344:101::b]:41474","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":107,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:06.635589698Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"4b3cc688-7368-44df-b2b6-1feae0cde60b","remote_addr":"[fd00:1122:3344:101::b]:41474","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":146,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:06.680522165Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:54994"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:06.680626427Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"89855451-976f-4936-b668-941b76816341","remote_addr":"[fd00:1122:3344:101::a]:54994","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":44,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:06.688244038Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"83ec96a1-396e-456c-9664-5dfefa6656cd","remote_addr":"[fd00:1122:3344:101::a]:54994","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":87,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:06.7196383Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:35094"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:06.721699603Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"4c565de5-c0ef-40f3-9e16-487a4a0fad2b","remote_addr":"[fd00:1122:3344:101::b]:35094","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":2005,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:06.858793732Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:63677"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:06.858991745Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"9a08afb9-219e-44bd-9ccb-26322b2315c4","remote_addr":"[fd00:1122:3344:101::a]:63677","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":126,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:06.983545074Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:57815"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:07.044851892Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"1b649b36-4858-48bf-8afe-9174026cc88c","remote_addr":"[fd00:1122:3344:101::b]:57815","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":61249,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:07.044967385Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:43104"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:07.090694424Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"d18fe9aa-8854-4730-8169-0e530d414537","remote_addr":"[fd00:1122:3344:101::a]:43104","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":45664,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:07.090914698Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"3c547750-f612-4fd7-936f-7751e0808b3b","remote_addr":"[fd00:1122:3344:101::b]:57815","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":105,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:07.091171312Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"7c0d2f1d-aa04-4a88-9d95-17478da8e1a4","remote_addr":"[fd00:1122:3344:101::a]:43104","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":143,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:21.435773293Z","hostname":"sock","pid":1555,"uri":"/4d88c6ce-c5a2-418b-b915-09fea24f811c","method":"GET","req_id":"6021c366-b2ca-499e-9b5c-a4422c757afe","remote_addr":"[fd00:1122:3344:101::d]:58565","local_addr":"[fd00:1122:3344:101::1]:56314","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":1791,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:24.367512175Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:46166"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:24.367570785Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"1ab4ba8d-4d3e-49dc-b79d-2df93008617f","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:46166","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":32,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:35.969552315Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:44477"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:35.969683267Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"0ad7125c-a4e8-4458-a40c-96ff261b2d12","remote_addr":"[fd00:1122:3344:101::c]:44477","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":98,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:35.977037265Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"77c40dda-3db4-4614-b6ca-2601e7319f25","remote_addr":"[fd00:1122:3344:101::c]:44477","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":131,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:35.993559049Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:38284"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:35.994498944Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"d3f0c977-229f-42ca-89cc-45229aa16ac7","remote_addr":"[fd00:1122:3344:101::c]:38284","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":889,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:36.151629213Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:62615"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:36.201141723Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"39e529d9-9951-4503-8429-4e9c4f052e78","remote_addr":"[fd00:1122:3344:101::c]:62615","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":49463,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:36.201401777Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"d3b0e577-c792-4252-89d1-798140e945cf","remote_addr":"[fd00:1122:3344:101::c]:62615","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":97,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:36.511747432Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:63498"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:36.511878614Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"0ac210bb-da43-46f3-8147-1da56ecaa87b","remote_addr":"[fd00:1122:3344:101::b]:63498","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":85,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:36.519096369Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"90b06167-163c-4163-aa46-a93384408c9b","remote_addr":"[fd00:1122:3344:101::b]:63498","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":141,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:36.564684477Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:38006"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:36.5648384Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"6be3830f-1d1f-4e33-afc1-3be93ec7fda2","remote_addr":"[fd00:1122:3344:101::b]:38006","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":132,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:36.601230861Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:41234"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:36.601332933Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"a6e1e5ac-450b-4e94-bddf-e1cd31fed54d","remote_addr":"[fd00:1122:3344:101::a]:41234","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":35,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:36.608308844Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"f962333d-1a02-40cc-92f7-e379259e66dc","remote_addr":"[fd00:1122:3344:101::a]:41234","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":83,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:36.611000437Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:50281"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:36.611127789Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"0c8f1676-3bde-4da6-938d-8e167c5e7896","remote_addr":"[fd00:1122:3344:101::a]:50281","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":78,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:36.694024483Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:55763"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:36.742311404Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"4729c836-6f00-451e-b805-70a087398d16","remote_addr":"[fd00:1122:3344:101::b]:55763","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":48236,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:36.742432886Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:42464"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:36.787706719Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"b9701dc6-ab64-49d9-ae91-7fdb5cd60d18","remote_addr":"[fd00:1122:3344:101::a]:42464","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":45204,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:36.787941172Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"f0e7f75b-c94a-4d90-b21c-547bd06e36ac","remote_addr":"[fd00:1122:3344:101::b]:55763","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":114,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:36.788179906Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"8b967fe2-20eb-4fe1-9c05-10f2ca477a28","remote_addr":"[fd00:1122:3344:101::a]:42464","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":158,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:37.462644846Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:34100"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:37.462857019Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"2d928026-85e2-4ac3-a3c6-a55a56f3bc37","remote_addr":"[fd00:1122:3344:101::a]:34100","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":192,"response_code":"200"} {"msg":"Mapping virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:37.465816866Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:832","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 253, 1, 91])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(14109874) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:37.465859937Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"PUT","req_id":"251a234c-ccb2-49a6-afed-f888ffb739d3","remote_addr":"[fd00:1122:3344:101::a]:34100","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":2824,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:37.618738458Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:43270"} {"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:37.650095559Z","hostname":"sock","pid":1555,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:548","metadata":"InstanceMetadata { silo_id: ab7e73b3-9e83-4cc6-b7be-0c5757fdaf0c, project_id: aac9ad5c-0170-42a6-930f-2f05e44928f2 }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-07-06T00:14:37.143291Z }","instance_runtime":"InstanceRuntimeState { propolis_id: Some(7243498a-d18c-4720-aa3a-10d138309072 (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-07-06T00:14:37.202713065Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 45a81b2f-856b-4af4-8146-33f69e6d0702, kind: Instance { id: 415a700f-113f-4ad3-9802-e6737b3e2041 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 253, 1, 91])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(14109874), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 45a81b2f-856b-4af4-8146-33f69e6d0702, kind: Instance { id: 415a700f-113f-4ad3-9802-e6737b3e2041 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 253, 1, 91])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(14109874), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 45a81b2f-856b-4af4-8146-33f69e6d0702, kind: Instance { id: 415a700f-113f-4ad3-9802-e6737b3e2041 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 253, 1, 91])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(14109874), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(14109874))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 45a81b2f-856b-4af4-8146-33f69e6d0702, kind: Instance { id: 415a700f-113f-4ad3-9802-e6737b3e2041 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 253, 1, 91])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(14109874), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-e32a9e62137\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 6b05bd11-5d10-4e2f-bcb8-d9867d31bb05, read_only_parent: Some(Volume { block_size: 512, id: 0e048977-70c5-4d69-9168-084b534a8562, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: ac804b16-9aa9-45b5-8578-d397ddb68f4a, key: Some(\"9LRd/jci9GherIoyUtbS0P8e6haJJCm1usQB8IrG7rY=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::14]:19002\", \"[fd00:1122:3344:101::17]:19001\", \"[fd00:1122:3344:101::1a]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 6b05bd11-5d10-4e2f-bcb8-d9867d31bb05, key: Some(\"E+itWzf7GJ8f+CqQ0ZmryvrCnYNqEERWmsUHNoXiznw=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::16]:19000\", \"[fd00:1122:3344:101::13]:19000\", \"[fd00:1122:3344:101::12]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAACd0gHmWOZYAADSAeZYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAntIB5ljmWAAA0gHmWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI0MTVhNzAwZi0xMTNmLTRhZDMtOTgwMi1lNjczN2IzZTIwNDEiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJRnQ2Qkpzbk9zU3Vudk9BWjJJN2dZQXBPMXpsQlcrQXZrdmx3UjJJcm9QKyJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }","propolis_id":"7243498a-d18c-4720-aa3a-10d138309072","instance_id":"415a700f-113f-4ad3-9802-e6737b3e2041"} {"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:37.65019747Z","hostname":"sock","pid":1555,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:582","instance_id":"415a700f-113f-4ad3-9802-e6737b3e2041 (instance)"} {"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:37.651653874Z","hostname":"sock","pid":1555,"component":"InstanceManager","file":"sled-agent/src/instance.rs:942","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 45a81b2f-856b-4af4-8146-33f69e6d0702, kind: Instance { id: 415a700f-113f-4ad3-9802-e6737b3e2041 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 253, 1, 91])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(14109874), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 45a81b2f-856b-4af4-8146-33f69e6d0702, kind: Instance { id: 415a700f-113f-4ad3-9802-e6737b3e2041 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 253, 1, 91])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(14109874), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 45a81b2f-856b-4af4-8146-33f69e6d0702, kind: Instance { id: 415a700f-113f-4ad3-9802-e6737b3e2041 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 253, 1, 91])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(14109874), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(14109874))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 45a81b2f-856b-4af4-8146-33f69e6d0702, kind: Instance { id: 415a700f-113f-4ad3-9802-e6737b3e2041 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 253, 1, 91])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(14109874), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-e32a9e62137\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 6b05bd11-5d10-4e2f-bcb8-d9867d31bb05, read_only_parent: Some(Volume { block_size: 512, id: 0e048977-70c5-4d69-9168-084b534a8562, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: ac804b16-9aa9-45b5-8578-d397ddb68f4a, key: Some(\"9LRd/jci9GherIoyUtbS0P8e6haJJCm1usQB8IrG7rY=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::14]:19002\", \"[fd00:1122:3344:101::17]:19001\", \"[fd00:1122:3344:101::1a]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 6b05bd11-5d10-4e2f-bcb8-d9867d31bb05, key: Some(\"E+itWzf7GJ8f+CqQ0ZmryvrCnYNqEERWmsUHNoXiznw=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::16]:19000\", \"[fd00:1122:3344:101::13]:19000\", \"[fd00:1122:3344:101::12]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAACd0gHmWOZYAADSAeZYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAntIB5ljmWAAA0gHmWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI0MTVhNzAwZi0xMTNmLTRhZDMtOTgwMi1lNjczN2IzZTIwNDEiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJRnQ2Qkpzbk9zU3Vudk9BWjJJN2dZQXBPMXpsQlcrQXZrdmx3UjJJcm9QKyJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }, instance_runtime: InstanceRuntimeState { propolis_id: Some(7243498a-d18c-4720-aa3a-10d138309072 (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-07-06T00:14:37.202713065Z }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-07-06T00:14:37.143291Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400 }","propolis_id":"7243498a-d18c-4720-aa3a-10d138309072","instance_id":"415a700f-113f-4ad3-9802-e6737b3e2041"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:37.654489359Z","hostname":"sock","pid":1555,"uri":"/instances/415a700f-113f-4ad3-9802-e6737b3e2041","method":"PUT","req_id":"dbd5dd66-65b2-4b88-b65f-c920a6c182f5","remote_addr":"[fd00:1122:3344:101::a]:43270","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":35634,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:37.80422644Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:51767"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:37.806187881Z","hostname":"sock","pid":1555,"uri":"/instances/415a700f-113f-4ad3-9802-e6737b3e2041/state","method":"PUT","req_id":"81561b35-ccd9-4e5d-834a-4a1d4e39ffcc","remote_addr":"[fd00:1122:3344:101::a]:51767","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":1851,"response_code":"200"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:37.836985943Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:478","port":"Port { inner: PortInner(PortData { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 253, 1, 91]), slot: 0, vni: Vni { inner: 14109874 }, subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 }, vnic: \"vopte6\" }) }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:37.837088575Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Configuring new Omicron zone: oxz_propolis-server_7243498a-d18c-4720-aa3a-10d138309072","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:37.870659161Z","hostname":"sock","pid":1555,"instance_id":"415a700f-113f-4ad3-9802-e6737b3e2041","component":"InstanceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_propolis-server_7243498a-d18c-4720-aa3a-10d138309072","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:37.893000127Z","hostname":"sock","pid":1555,"instance_id":"415a700f-113f-4ad3-9802-e6737b3e2041","component":"InstanceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_propolis-server_7243498a-d18c-4720-aa3a-10d138309072:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:39.553771055Z","hostname":"sock","pid":1555,"instance_id":"415a700f-113f-4ad3-9802-e6737b3e2041","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:39.555088096Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_7243498a-d18c-4720-aa3a-10d138309072","instance_id":"415a700f-113f-4ad3-9802-e6737b3e2041","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_propolis-server_7243498a-d18c-4720-aa3a-10d138309072\") failed: Property not found. retry in 58.665659ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:14:39.73944416Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_7243498a-d18c-4720-aa3a-10d138309072","instance_id":"415a700f-113f-4ad3-9802-e6737b3e2041","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_7243498a-d18c-4720-aa3a-10d138309072\") failed: Property not found. retry in 137.358704ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:14:39.809779893Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_7243498a-d18c-4720-aa3a-10d138309072","instance_id":"415a700f-113f-4ad3-9802-e6737b3e2041","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_7243498a-d18c-4720-aa3a-10d138309072\") failed: Property not found. retry in 139.042727ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:14:39.971461274Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_7243498a-d18c-4720-aa3a-10d138309072","instance_id":"415a700f-113f-4ad3-9802-e6737b3e2041","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_7243498a-d18c-4720-aa3a-10d138309072\") failed: Property not found. retry in 574.223333ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:14:40.124758572Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_7243498a-d18c-4720-aa3a-10d138309072","instance_id":"415a700f-113f-4ad3-9802-e6737b3e2041","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_7243498a-d18c-4720-aa3a-10d138309072\") failed: Property not found. retry in 997.204357ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:14:40.713564863Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_7243498a-d18c-4720-aa3a-10d138309072","instance_id":"415a700f-113f-4ad3-9802-e6737b3e2041","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_7243498a-d18c-4720-aa3a-10d138309072\") failed: Property not found. retry in 1.199693316s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:14:41.721998496Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_7243498a-d18c-4720-aa3a-10d138309072","instance_id":"415a700f-113f-4ad3-9802-e6737b3e2041","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_7243498a-d18c-4720-aa3a-10d138309072\") failed: Property not found. retry in 1.438661811s","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:14:42.93226969Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_7243498a-d18c-4720-aa3a-10d138309072","instance_id":"415a700f-113f-4ad3-9802-e6737b3e2041","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_7243498a-d18c-4720-aa3a-10d138309072\") failed: Property not found. retry in 932.775545ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:14:44.381288956Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_7243498a-d18c-4720-aa3a-10d138309072","instance_id":"415a700f-113f-4ad3-9802-e6737b3e2041","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Started propolis in zone: oxz_propolis-server_7243498a-d18c-4720-aa3a-10d138309072","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:45.34727433Z","hostname":"sock","pid":1555,"instance_id":"415a700f-113f-4ad3-9802-e6737b3e2041","component":"InstanceManager","file":"sled-agent/src/instance.rs:1428"} {"msg":"wait for service Some(\"oxz_propolis-server_7243498a-d18c-4720-aa3a-10d138309072\") failed: Property not found. retry in 62.380766ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:14:45.355904798Z","hostname":"sock","pid":1555,"instance_id":"415a700f-113f-4ad3-9802-e6737b3e2041","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_7243498a-d18c-4720-aa3a-10d138309072\") failed: Property not found. retry in 117.20487ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:14:45.427824917Z","hostname":"sock","pid":1555,"instance_id":"415a700f-113f-4ad3-9802-e6737b3e2041","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_7243498a-d18c-4720-aa3a-10d138309072\") failed: Property not found. retry in 128.118388ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:14:45.555303632Z","hostname":"sock","pid":1555,"instance_id":"415a700f-113f-4ad3-9802-e6737b3e2041","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_7243498a-d18c-4720-aa3a-10d138309072\") failed: Property not found. retry in 550.699068ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:14:45.692233379Z","hostname":"sock","pid":1555,"instance_id":"415a700f-113f-4ad3-9802-e6737b3e2041","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:46.253793545Z","hostname":"sock","pid":1555,"instance_id":"415a700f-113f-4ad3-9802-e6737b3e2041","component":"InstanceManager","file":"sled-agent/src/instance.rs:1437"} {"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:46.304477985Z","hostname":"sock","pid":1555,"instance_id":"415a700f-113f-4ad3-9802-e6737b3e2041","component":"InstanceManager","file":"sled-agent/src/instance.rs:1451"} {"msg":"Sending ensure request to propolis: InstanceEnsureRequest { cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAACd0gHmWOZYAADSAeZYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAntIB5ljmWAAA0gHmWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI0MTVhNzAwZi0xMTNmLTRhZDMtOTgwMi1lNjczN2IzZTIwNDEiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJRnQ2Qkpzbk9zU3Vudk9BWjJJN2dZQXBPMXpsQlcrQXZrdmx3UjJJcm9QKyJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"), disks: [DiskRequest { device: \"nvme\", name: \"disk-e32a9e62137\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 6b05bd11-5d10-4e2f-bcb8-d9867d31bb05, read_only_parent: Some(Volume { block_size: 512, id: 0e048977-70c5-4d69-9168-084b534a8562, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: ac804b16-9aa9-45b5-8578-d397ddb68f4a, key: Some(\"9LRd/jci9GherIoyUtbS0P8e6haJJCm1usQB8IrG7rY=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::14]:19002\", \"[fd00:1122:3344:101::17]:19001\", \"[fd00:1122:3344:101::1a]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 6b05bd11-5d10-4e2f-bcb8-d9867d31bb05, key: Some(\"E+itWzf7GJ8f+CqQ0ZmryvrCnYNqEERWmsUHNoXiznw=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::16]:19000\", \"[fd00:1122:3344:101::13]:19000\", \"[fd00:1122:3344:101::12]:19000\"] } }] } }], migrate: None, nics: [NetworkInterfaceRequest { name: \"vopte6\", slot: Slot(0) }], properties: InstanceProperties { bootrom_id: 00000000-0000-0000-0000-000000000000, description: \"Test description\", id: 415a700f-113f-4ad3-9802-e6737b3e2041, image_id: 00000000-0000-0000-0000-000000000000, memory: 1024, metadata: InstanceMetadata { project_id: aac9ad5c-0170-42a6-930f-2f05e44928f2, silo_id: ab7e73b3-9e83-4cc6-b7be-0c5757fdaf0c }, name: \"localshark\", vcpus: 2 } }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:46.313128573Z","hostname":"sock","pid":1555,"instance_id":"415a700f-113f-4ad3-9802-e6737b3e2041","component":"InstanceManager","file":"sled-agent/src/instance.rs:689"} {"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:46.499403286Z","hostname":"sock","pid":1555,"instance_id":"415a700f-113f-4ad3-9802-e6737b3e2041","component":"InstanceManager","file":"sled-agent/src/instance.rs:691"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Starting), migration_status: NoMigration, time: 2024-07-06T00:14:46.500813939Z }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:46.500967871Z","hostname":"sock","pid":1555,"instance_id":"415a700f-113f-4ad3-9802-e6737b3e2041","component":"InstanceManager","file":"sled-agent/src/instance.rs:573"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:46.500993372Z","hostname":"sock","pid":1555,"instance_id":"415a700f-113f-4ad3-9802-e6737b3e2041","component":"InstanceManager","file":"sled-agent/src/instance.rs:595","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-07-06T00:14:46.500813939Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(7243498a-d18c-4720-aa3a-10d138309072 (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-07-06T00:14:37.202713065Z }","propolis_id":"7243498a-d18c-4720-aa3a-10d138309072"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:46.501020712Z","hostname":"sock","pid":1555,"instance_id":"415a700f-113f-4ad3-9802-e6737b3e2041","component":"InstanceManager","file":"sled-agent/src/instance.rs:492","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(7243498a-d18c-4720-aa3a-10d138309072 (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-07-06T00:14:37.202713065Z }, propolis_id: 7243498a-d18c-4720-aa3a-10d138309072 (propolis), vmm_state: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-07-06T00:14:46.500813939Z }, migration_state: None }","instance_id":"415a700f-113f-4ad3-9802-e6737b3e2041"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_status: NoMigration, time: 2024-07-06T00:14:46.598644291Z }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:46.598663982Z","hostname":"sock","pid":1555,"instance_id":"415a700f-113f-4ad3-9802-e6737b3e2041","component":"InstanceManager","file":"sled-agent/src/instance.rs:573"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:46.598687632Z","hostname":"sock","pid":1555,"instance_id":"415a700f-113f-4ad3-9802-e6737b3e2041","component":"InstanceManager","file":"sled-agent/src/instance.rs:595","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-07-06T00:14:46.598644291Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(7243498a-d18c-4720-aa3a-10d138309072 (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-07-06T00:14:37.202713065Z }","propolis_id":"7243498a-d18c-4720-aa3a-10d138309072"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:46.598730932Z","hostname":"sock","pid":1555,"instance_id":"415a700f-113f-4ad3-9802-e6737b3e2041","component":"InstanceManager","file":"sled-agent/src/instance.rs:492","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(7243498a-d18c-4720-aa3a-10d138309072 (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-07-06T00:14:37.202713065Z }, propolis_id: 7243498a-d18c-4720-aa3a-10d138309072 (propolis), vmm_state: VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-07-06T00:14:46.598644291Z }, migration_state: None }","instance_id":"415a700f-113f-4ad3-9802-e6737b3e2041"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:46.635432329Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:43283"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:46.635617852Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"a82393ef-1ff7-423f-bdfc-9dae7116ba8e","remote_addr":"[fd00:1122:3344:101::b]:43283","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":129,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:46.682682473Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"cbe45643-74d0-4d31-ae74-806a90abe92f","remote_addr":"[fd00:1122:3344:101::b]:43283","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":1164,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:46.723016007Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:33010"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:46.723071288Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"3a41f78e-d5ba-4755-851b-c20cc8ad8822","remote_addr":"[fd00:1122:3344:101::b]:33010","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":38,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:46.734360898Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"43eff213-d66d-44a9-9a5a-f5d558bff508","remote_addr":"[fd00:1122:3344:101::b]:33010","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":88,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:51.438380527Z","hostname":"sock","pid":1555,"uri":"/4d88c6ce-c5a2-418b-b915-09fea24f811c","method":"GET","req_id":"cc222013-193c-422f-9e8d-8fb34c682b68","remote_addr":"[fd00:1122:3344:101::d]:58565","local_addr":"[fd00:1122:3344:101::1]:56314","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"4d88c6ce-c5a2-418b-b915-09fea24f811c","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":4314,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:54.399968975Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:37284"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:14:54.400028436Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"776121df-0d4c-48f3-82cd-ba28cfb91403","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:37284","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":40,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:00.608875583Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:51026"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:00.609568064Z","hostname":"sock","pid":1555,"uri":"/instances/415a700f-113f-4ad3-9802-e6737b3e2041/state","method":"PUT","req_id":"156f89b8-632d-43fa-9a20-782f8f999aaf","remote_addr":"[fd00:1122:3344:101::a]:51026","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":622,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_status: NoMigration, time: 2024-07-06T00:15:00.609699286Z }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:00.609741577Z","hostname":"sock","pid":1555,"instance_id":"415a700f-113f-4ad3-9802-e6737b3e2041","component":"InstanceManager","file":"sled-agent/src/instance.rs:573"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:00.609781247Z","hostname":"sock","pid":1555,"instance_id":"415a700f-113f-4ad3-9802-e6737b3e2041","component":"InstanceManager","file":"sled-agent/src/instance.rs:595","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-07-06T00:15:00.609699286Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(7243498a-d18c-4720-aa3a-10d138309072 (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-07-06T00:14:37.202713065Z }","propolis_id":"7243498a-d18c-4720-aa3a-10d138309072"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:00.609809248Z","hostname":"sock","pid":1555,"instance_id":"415a700f-113f-4ad3-9802-e6737b3e2041","component":"InstanceManager","file":"sled-agent/src/instance.rs:492","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(7243498a-d18c-4720-aa3a-10d138309072 (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-07-06T00:14:37.202713065Z }, propolis_id: 7243498a-d18c-4720-aa3a-10d138309072 (propolis), vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-07-06T00:15:00.609699286Z }, migration_state: None }","instance_id":"415a700f-113f-4ad3-9802-e6737b3e2041"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_status: NoMigration, time: 2024-07-06T00:15:00.702444157Z }","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:00.702524948Z","hostname":"sock","pid":1555,"instance_id":"415a700f-113f-4ad3-9802-e6737b3e2041","component":"InstanceManager","file":"sled-agent/src/instance.rs:573"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:00.70261413Z","hostname":"sock","pid":1555,"instance_id":"415a700f-113f-4ad3-9802-e6737b3e2041","component":"InstanceManager","file":"sled-agent/src/instance.rs:595","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(6), time_updated: 2024-07-06T00:15:00.702444157Z }","new_instance_state":"InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-07-06T00:15:00.702444157Z }","propolis_id":"7243498a-d18c-4720-aa3a-10d138309072"} {"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:00.70264449Z","hostname":"sock","pid":1555,"instance_id":"415a700f-113f-4ad3-9802-e6737b3e2041","component":"InstanceManager","file":"sled-agent/src/instance.rs:610","instance_id":"415a700f-113f-4ad3-9802-e6737b3e2041"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:00.70265921Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:00.702867434Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:406"} {"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:00.702928835Z","hostname":"sock","pid":1555,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:437","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/f8eb5131-e7a0-4521-9f31-d994085bd770/debug/bundle/zone\", \"/pool/int/fea674a6-6f32-4040-ae6a-1581a3ccf192/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/9e1edfda-939b-489d-a7cb-446b4599864c/crypt/debug\", \"/pool/ext/c029673f-d8f0-4d66-9f90-dcc8d5377a3a/crypt/debug\", \"/pool/ext/187c3b21-21e5-4726-9974-0290b3e4dbd4/crypt/debug\", \"/pool/ext/69edb71d-4c73-4b0c-a984-e973915bfc3d/crypt/debug\", \"/pool/ext/05665420-5e42-4534-83d1-b4b73f92c491/crypt/debug\", \"/pool/ext/1fb8d37a-fccb-4d16-a337-6873d3def608/crypt/debug\", \"/pool/ext/03b41f7a-86c6-414e-8276-ffa92bf542a8/crypt/debug\", \"/pool/ext/d0e51e3d-0355-4324-9d0e-c910d0a56ce1/crypt/debug\", \"/pool/ext/af3bec9c-e31f-4896-95d7-21d956ea9d1e/crypt/debug\"] }","zone_name":"oxz_propolis-server_7243498a-d18c-4720-aa3a-10d138309072"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:00.752420585Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:47069"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:00.753521973Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"182eecf3-0d98-4afe-892d-f0618d1ae1b1","remote_addr":"[fd00:1122:3344:101::b]:47069","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":981,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:00.769326595Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"b174038e-b81d-4963-ba02-98076a79d5ec","remote_addr":"[fd00:1122:3344:101::b]:47069","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":125,"response_code":"204"} {"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:01.509445572Z","hostname":"sock","pid":1555,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1230","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_7243498a-d18c-4720-aa3a-10d138309072\", bundle_id: a79bf5d4-3418-49e5-9b41-afcb3460889c }, time_created: 2024-07-06T00:15:00.704254056Z, version: 0, cause: TerminatedInstance }"} {"msg":"Halting and removing zone: oxz_propolis-server_7243498a-d18c-4720-aa3a-10d138309072","v":0,"name":"SledAgent","level":40,"time":"2024-07-06T00:15:01.509559174Z","hostname":"sock","pid":1555,"instance_id":"415a700f-113f-4ad3-9802-e6737b3e2041","component":"InstanceManager","file":"sled-agent/src/instance.rs:784"} {"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:03.297255748Z","hostname":"sock","pid":1555,"instance_id":"415a700f-113f-4ad3-9802-e6737b3e2041","component":"InstanceManager","file":"illumos-utils/src/zone.rs:269"} {"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:03.312509822Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:958","id":"RouterId { vni: Vni(14109874), kind: System }"} {"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:03.312562613Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:958","id":"RouterId { vni: Vni(14109874), kind: Custom(V4(Ipv4Net { addr: 172.30.0.0, width: 22 })) }"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:03.321374813Z","hostname":"sock","pid":1555,"instance_id":"415a700f-113f-4ad3-9802-e6737b3e2041","component":"InstanceManager","file":"sled-agent/src/instance.rs:492","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-07-06T00:15:00.702444157Z }, propolis_id: 7243498a-d18c-4720-aa3a-10d138309072 (propolis), vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-07-06T00:15:03.321314452Z }, migration_state: None }","instance_id":"415a700f-113f-4ad3-9802-e6737b3e2041"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:03.443858268Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:47912"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:03.444049141Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"4fbb8e37-fc14-4667-83f1-efed07a19bf0","remote_addr":"[fd00:1122:3344:101::b]:47912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":169,"response_code":"200"} {"msg":"Mapping virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:03.444304045Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:832","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 253, 1, 91])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(14109874) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:03.444337856Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"PUT","req_id":"b6670850-7abd-4175-b209-7b40e12cfe7e","remote_addr":"[fd00:1122:3344:101::b]:47912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":103,"response_code":"204"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:03.674896287Z","hostname":"sock","pid":1555,"instance_id":"415a700f-113f-4ad3-9802-e6737b3e2041","component":"InstanceManager","file":"sled-agent/src/instance.rs:492","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-07-06T00:15:00.702444157Z }, propolis_id: 7243498a-d18c-4720-aa3a-10d138309072 (propolis), vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-07-06T00:15:03.321314452Z }, migration_state: None }","instance_id":"415a700f-113f-4ad3-9802-e6737b3e2041"} {"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:03.678483475Z","hostname":"sock","pid":1555,"instance_id":"415a700f-113f-4ad3-9802-e6737b3e2041","component":"InstanceManager","file":"sled-agent/src/instance.rs:727"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:03.711835817Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:43360"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:03.711923259Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"8b967821-1c50-46a5-9217-b0722dc3f602","remote_addr":"[fd00:1122:3344:101::b]:43360","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":43,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:03.718510294Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"1231074d-0ee4-44ca-8e7a-dfaee080572e","remote_addr":"[fd00:1122:3344:101::b]:43360","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":113,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:03.755551415Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:64586"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:03.755700208Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"00c043e3-b758-4de4-a4f5-397dfb8076dd","remote_addr":"[fd00:1122:3344:101::b]:64586","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":103,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:03.762504677Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"ca607a52-c695-4285-874d-7385029cc366","remote_addr":"[fd00:1122:3344:101::b]:64586","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":837,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:05.100934517Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:48705"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:05.101184792Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"5690e910-f564-4dd2-87ee-23da8bb979ec","remote_addr":"[fd00:1122:3344:101::a]:48705","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":230,"response_code":"200"} {"msg":"Clearing mapping of virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:05.102115456Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:872","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 253, 1, 91])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(14109874) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:05.102133957Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"DELETE","req_id":"aa0d9198-ed28-426f-942c-fde491f03d02","remote_addr":"[fd00:1122:3344:101::a]:48705","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":774,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:05.946023051Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:47561"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:05.946166003Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"b34ce46a-ca4a-45ba-8337-b7d4d1bcf955","remote_addr":"[fd00:1122:3344:101::c]:47561","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":128,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:05.957454463Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"37cc8e29-757a-480f-a32b-e438e8bcb183","remote_addr":"[fd00:1122:3344:101::c]:47561","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":141,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:05.964001168Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:60029"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:05.965023594Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"b2e36954-8ff1-413a-a440-f729befff52e","remote_addr":"[fd00:1122:3344:101::c]:60029","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":999,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:06.329958061Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:48779"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:06.381826069Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"45257978-0d1b-4a01-80d0-5108114cf836","remote_addr":"[fd00:1122:3344:101::c]:48779","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":51802,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:06.382084784Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"b0b37608-c3a2-4824-81c1-6c47defd20de","remote_addr":"[fd00:1122:3344:101::c]:48779","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":92,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:06.671292991Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:44083"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:06.671440824Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"26182688-ea17-48df-85c9-c0bb0ffb7165","remote_addr":"[fd00:1122:3344:101::b]:44083","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":98,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:06.677596832Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:59286"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:06.677803155Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"66cf5024-38b8-4999-a929-43854424a441","remote_addr":"[fd00:1122:3344:101::b]:59286","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":170,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:06.680047331Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"d50fb79f-2e92-4f16-a717-41fd240a9b72","remote_addr":"[fd00:1122:3344:101::b]:44083","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":113,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:06.714448301Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:53543"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:06.714604113Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"ccf58673-2e83-4331-9d75-aacff0c0ef85","remote_addr":"[fd00:1122:3344:101::a]:53543","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":86,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:06.822819271Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:50405"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:06.822971464Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"cf9ec4bf-5a41-4d85-81ec-2aa7f3f11dba","remote_addr":"[fd00:1122:3344:101::a]:50405","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":109,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:06.830446303Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"802a0d95-49b7-4bbc-b268-55a8f4905bd2","remote_addr":"[fd00:1122:3344:101::a]:50405","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":125,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:06.876493498Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:47666"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:06.9261178Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"d0a70221-a356-4073-840b-e4310eacfa17","remote_addr":"[fd00:1122:3344:101::b]:47666","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":49585,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:06.926353754Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"90b82d76-b078-4917-a1c9-37f6d679e7ab","remote_addr":"[fd00:1122:3344:101::b]:47666","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":85,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:07.043360873Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:46629"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:07.043780849Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"d50d9658-2917-43de-8eb0-ec188b5ef34d","remote_addr":"[fd00:1122:3344:101::a]:46629","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":367,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:07.052389097Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"2509dd3a-316a-47ac-a3f9-07dca2322b34","remote_addr":"[fd00:1122:3344:101::a]:46629","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":120,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:07.078439763Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:37806"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:07.124937765Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"474b8679-8d88-4848-92b7-4177917647be","remote_addr":"[fd00:1122:3344:101::a]:37806","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":46456,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-06T00:15:07.12522237Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"5ec65014-141c-4734-ab64-4c21aab18420","remote_addr":"[fd00:1122:3344:101::a]:37806","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/6a3f84c/dropshot/src/server.rs:878","latency_us":108,"response_code":"200"}